feat(appliance): backport all recent appliance changes (#64182)

Draft in case plan in https://linear.app/sourcegraph/issue/REL-309/release-process-for-appliance not agreed. Please see that first. Generated by: ``` git log --format=%H d47b4cc..origin/main -- cmd/appliance internal/appliance docker-images/appliance-frontend | tac | xargs git cherry-pick ``` d47b4cc being the commit we branched off main from to create the 5.5.x branch (https://buildkite.com/sourcegraph/sourcegraph/builds/281882). Commits (generated by `git log --format='- https://github.com/sourcegraph/sourcegraph/commit/%H' d47b4cc..origin/main -- cmd/appliance internal/appliance docker-images/appliance-frontend | tac`): - https://github.com/sourcegraph/sourcegraph/commit/a20b0650b453d11d7dc428b9f919b79465db088c - https://github.com/sourcegraph/sourcegraph/commit/b71c986c77829cfe3bdc6f2d31e8590fdb969b17 - https://github.com/sourcegraph/sourcegraph/commit/91864283bc0811928188f415e6cda7d3fe940e34 - https://github.com/sourcegraph/sourcegraph/commit/c88b57020fa49fe996e37ca9409a541d48d7a63e - https://github.com/sourcegraph/sourcegraph/commit/0491839942c3087687d0f48ade1c544f1fbcb881 - https://github.com/sourcegraph/sourcegraph/commit/619fc5707415bd50631153bcb1b01c670d44fd82 - https://github.com/sourcegraph/sourcegraph/commit/e81c39a834810a60c8827456963511213eb5f602 - https://github.com/sourcegraph/sourcegraph/commit/a61f353e0e93f848fa476ffda6d46f71060d465d - https://github.com/sourcegraph/sourcegraph/commit/0abef7b43d5dadac92a3ff318f3caf6d7b572f6a - https://github.com/sourcegraph/sourcegraph/commit/0e391a964ae8d21904bd875862759db521feb5ba - https://github.com/sourcegraph/sourcegraph/commit/daae9adfb642f2c73e7b4b2dcb7acca618ae86d3 - https://github.com/sourcegraph/sourcegraph/commit/6e31f0f4cc89ad7d3892e67bad55ab9a0b9658a7 - https://github.com/sourcegraph/sourcegraph/commit/49a600220d5eb2875ab54c87329d0a90f15c38d6 - https://github.com/sourcegraph/sourcegraph/commit/37cf4a7b7e9dfb4f09635d2e04ce30daf5d8b7a0 - https://github.com/sourcegraph/sourcegraph/commit/29fc613c376fe38055fa907c5bbd293ea6e4408f - https://github.com/sourcegraph/sourcegraph/commit/255e6387cc7854c8aed6dfa1445fcade380ec690 - https://github.com/sourcegraph/sourcegraph/commit/49b32fcf3a1ec8362e3be286de182aba76f93917 - https://github.com/sourcegraph/sourcegraph/commit/9f4c160f91838ed3167a38ba62b7ca90ad90a771 - https://github.com/sourcegraph/sourcegraph/commit/3814fd7390455354be1129cdb1b240c201ef5964 - https://github.com/sourcegraph/sourcegraph/commit/c68e92bc28b3ce0878aebe8e10471cca89010b77 - https://github.com/sourcegraph/sourcegraph/commit/7e82c27ab5de1c76f1e877ec9dee665800807cbb - https://github.com/sourcegraph/sourcegraph/commit/98c6b9703f889bb8b706855204276d3528b03356 - https://github.com/sourcegraph/sourcegraph/commit/a01ebad8417d4c14b969f61a3a6ad8e836728047 - https://github.com/sourcegraph/sourcegraph/commit/8c2d8da234cc46b529b8df92a38cdf7f7d83a7a7 - https://github.com/sourcegraph/sourcegraph/commit/ebec72d7ed7046406d22d2bf8276612945db358d - https://github.com/sourcegraph/sourcegraph/commit/d945f192852a7dc3cd2482602fe37f614f82f8a1 - https://github.com/sourcegraph/sourcegraph/commit/84e28998e9d3976b79d858439953d101ca44ab7e ## Test plan Tests pass. ## Changelog - Backport all recent appliance changes. The appliance is still pre-release. --------- Co-authored-by: Jacob Pleiness <[email protected]> Co-authored-by: Anish Lakhwara <[email protected]> Co-authored-by: Warren Gifford <[email protected]> Co-authored-by: Nelson Araujo <[email protected]>
sourcegraph · Jul 31, 2024 · d24e8fe · d24e8fe
1 parent 162d383
commit d24e8fe
Show file tree

Hide file tree

Showing 234 changed files with 8,985 additions and 1,844 deletions.
diff --git a/client/web/dev/utils/create-js-context.ts b/client/web/dev/utils/create-js-context.ts
@@ -35,6 +35,7 @@ export const createJsContext = ({ sourcegraphBaseUrl }: { sourcegraphBaseUrl: st
         accessTokensExpirationDaysOptions: [7, 14, 30, 60, 90],
         allowSignup: true,
         batchChangesEnabled: true,
+        applianceManaged: false,
         batchChangesDisableWebhooksWarning: false,
         batchChangesWebhookLogsEnabled: true,
         executorsEnabled: false,

diff --git a/client/web/src/enterprise/site-admin/SiteAdminSidebar.story.tsx b/client/web/src/enterprise/site-admin/SiteAdminSidebar.story.tsx
@@ -38,6 +38,7 @@ export const AdminSidebarItems: StoryFn = () => (
                     batchChangesExecutionEnabled={true}
                     batchChangesWebhookLogsEnabled={true}
                     codeInsightsEnabled={true}
+                    applianceManaged={false}
                     endUserOnboardingEnabled={false}
                 />
                 <SiteAdminSidebar
@@ -48,6 +49,7 @@ export const AdminSidebarItems: StoryFn = () => (
                     batchChangesExecutionEnabled={true}
                     batchChangesWebhookLogsEnabled={true}
                     codeInsightsEnabled={true}
+                    applianceManaged={false}
                     endUserOnboardingEnabled={false}
                 />
                 <SiteAdminSidebar
@@ -58,6 +60,7 @@ export const AdminSidebarItems: StoryFn = () => (
                     batchChangesExecutionEnabled={false}
                     batchChangesWebhookLogsEnabled={false}
                     codeInsightsEnabled={true}
+                    applianceManaged={false}
                     endUserOnboardingEnabled={false}
                 />
                 <SiteAdminSidebar
@@ -68,6 +71,7 @@ export const AdminSidebarItems: StoryFn = () => (
                     batchChangesExecutionEnabled={true}
                     batchChangesWebhookLogsEnabled={true}
                     codeInsightsEnabled={false}
+                    applianceManaged={false}
                     endUserOnboardingEnabled={false}
                 />
             </Grid>

diff --git a/client/web/src/integration/jscontext.ts b/client/web/src/integration/jscontext.ts
@@ -27,6 +27,7 @@ export const createJsContext = ({ sourcegraphBaseUrl }: { sourcegraphBaseUrl: st
     accessTokensExpirationDaysOptions: [7, 30, 60, 90],
     allowSignup: false,
     batchChangesEnabled: true,
+    applianceManaged: false,
     batchChangesDisableWebhooksWarning: false,
     batchChangesWebhookLogsEnabled: true,
     codeInsightsEnabled: true,

diff --git a/client/web/src/jscontext.ts b/client/web/src/jscontext.ts
@@ -196,6 +196,11 @@ export interface SourcegraphContext extends Pick<Required<SiteConfiguration>, 'e
 
     batchChangesWebhookLogsEnabled: boolean
 
+    /**
+     * Whether this sourcegraph instance is managed by Appliance
+     */
+    applianceManaged: boolean
+
     /**
      * Whether Cody is enabled on this instance. Check
      * {@link SourcegraphContext.codyEnabledForCurrentUser} to see whether Cody is enabled for the

diff --git a/client/web/src/routes.tsx b/client/web/src/routes.tsx
@@ -268,6 +268,7 @@ export const routes: RouteObject[] = [
                         sideBarGroups={props.siteAdminSideBarGroups}
                         overviewComponents={props.siteAdminOverviewComponents}
                         codeInsightsEnabled={window.context.codeInsightsEnabled}
+                        applianceManaged={window.context.applianceManaged}
                         telemetryRecorder={props.platformContext.telemetryRecorder}
                     />
                 )}

diff --git a/client/web/src/site-admin/SiteAdminArea.tsx b/client/web/src/site-admin/SiteAdminArea.tsx
@@ -59,6 +59,7 @@ export interface SiteAdminAreaRouteContext
     overviewComponents: readonly React.ComponentType<React.PropsWithChildren<{}>>[]
 
     codeInsightsEnabled: boolean
+    applianceManaged: boolean
 
     endUserOnboardingEnabled: boolean
 }
@@ -77,6 +78,7 @@ interface SiteAdminAreaProps
     authenticatedUser: AuthenticatedUser
     isSourcegraphDotCom: boolean
     codeInsightsEnabled: boolean
+    applianceManaged: boolean
 }
 
 const sourcegraphOperatorSiteAdminMaintenanceBlockItems = new Set([
@@ -142,6 +144,7 @@ const AuthenticatedSiteAdminArea: React.FunctionComponent<React.PropsWithChildre
         telemetryService: props.telemetryService,
         telemetryRecorder: props.telemetryRecorder,
         codeInsightsEnabled: props.codeInsightsEnabled,
+        applianceManaged: props.applianceManaged,
         endUserOnboardingEnabled,
     }
 
@@ -161,6 +164,7 @@ const AuthenticatedSiteAdminArea: React.FunctionComponent<React.PropsWithChildre
                     batchChangesExecutionEnabled={props.batchChangesExecutionEnabled}
                     batchChangesWebhookLogsEnabled={props.batchChangesWebhookLogsEnabled}
                     codeInsightsEnabled={props.codeInsightsEnabled}
+                    applianceManaged={props.applianceManaged}
                     endUserOnboardingEnabled={endUserOnboardingEnabled}
                 />
                 <div className="flex-bounded">

diff --git a/client/web/src/site-admin/SiteAdminSidebar.tsx b/client/web/src/site-admin/SiteAdminSidebar.tsx
@@ -15,6 +15,7 @@ export interface SiteAdminSideBarGroupContext extends BatchChangesProps {
     isSourcegraphDotCom: boolean
     codeInsightsEnabled: boolean
     endUserOnboardingEnabled: boolean
+    applianceManaged: boolean
 }
 
 export interface SiteAdminSideBarGroup extends NavGroupDescriptor<SiteAdminSideBarGroupContext> {}

diff --git a/client/web/src/site-admin/sidebaritems.ts b/client/web/src/site-admin/sidebaritems.ts
@@ -135,6 +135,12 @@ const maintenanceGroup: SiteAdminSideBarGroup = {
         {
             label: maintenanceGroupUpdatesItemLabel,
             to: '/site-admin/updates',
+            condition: ({ applianceManaged }) => !applianceManaged,
+        },
+        {
+            label: maintenanceGroupUpdatesItemLabel,
+            to: '/appliance/updates',
+            condition: ({ applianceManaged }) => applianceManaged,
         },
         {
             label: 'Documentation',

diff --git a/cmd/appliance/shared/BUILD.bazel b/cmd/appliance/shared/BUILD.bazel
@@ -11,7 +11,9 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//internal/appliance",
+        "//internal/appliance/healthchecker",
         "//internal/appliance/reconciler",
+        "//internal/appliance/selfupdate",
         "//internal/appliance/v1:appliance",
         "//internal/debugserver",
         "//internal/env",
@@ -23,6 +25,7 @@ go_library(
         "//lib/errors",
         "@com_github_sourcegraph_log//:log",
         "@com_github_sourcegraph_log_logr//:logr",
+        "@io_k8s_apimachinery//pkg/types",
         "@io_k8s_client_go//rest",
         "@io_k8s_client_go//tools/clientcmd",
         "@io_k8s_client_go//util/homedir",

diff --git a/cmd/appliance/shared/config.go b/cmd/appliance/shared/config.go
@@ -16,13 +16,15 @@ import (
 type Config struct {
 	env.BaseConfig
 
-	k8sConfig        *rest.Config
-	metrics          metricsConfig
-	grpc             grpcConfig
-	http             httpConfig
-	namespace        string
-	relregEndpoint   string
-	applianceVersion string
+	k8sConfig              *rest.Config
+	metrics                metricsConfig
+	grpc                   grpcConfig
+	http                   httpConfig
+	namespace              string
+	relregEndpoint         string
+	applianceVersion       string
+	selfDeploymentName     string
+	noResourceRestrictions string
 }
 
 func (c *Config) Load() {
@@ -43,10 +45,12 @@ func (c *Config) Load() {
 	c.metrics.addr = c.Get("APPLIANCE_METRICS_ADDR", ":8734", "Appliance metrics server address.")
 	c.metrics.secure = c.GetBool("APPLIANCE_METRICS_SECURE", "false", "Appliance metrics server uses https.")
 	c.grpc.addr = c.Get("APPLIANCE_GRPC_ADDR", ":9000", "Appliance gRPC address.")
-	c.http.addr = c.Get("APPLIANCE_HTTP_ADDR", ":8080", "Appliance http address.")
+	c.http.addr = c.Get("APPLIANCE_HTTP_ADDR", ":8888", "Appliance http address.")
 	c.namespace = c.Get("APPLIANCE_NAMESPACE", "default", "Namespace to monitor.")
 	c.applianceVersion = c.Get("APPLIANCE_VERSION", version.Version(), "Version tag for the running appliance.")
+	c.selfDeploymentName = c.Get("APPLIANCE_DEPLOYMENT_NAME", "", "Own deployment name for self-update. Default is to disable self-update.")
 	c.relregEndpoint = c.Get("RELEASE_REGISTRY_ENDPOINT", releaseregistry.Endpoint, "Release registry endpoint.")
+	c.noResourceRestrictions = c.Get("APPLIANCE_NO_RESOURCE_RESTRICTIONS", "false", "Remove all resource requests and limits from deployed resources. Only recommended for local development.")
 }
 
 func (c *Config) Validate() error {

diff --git a/cmd/appliance/shared/shared.go b/cmd/appliance/shared/shared.go
@@ -6,20 +6,25 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 	"time"
 
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
+	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"github.com/sourcegraph/log"
 	sglogr "github.com/sourcegraph/log/logr"
+
 	"github.com/sourcegraph/sourcegraph/internal/appliance"
+	"github.com/sourcegraph/sourcegraph/internal/appliance/healthchecker"
 	"github.com/sourcegraph/sourcegraph/internal/appliance/reconciler"
+	"github.com/sourcegraph/sourcegraph/internal/appliance/selfupdate"
 	pb "github.com/sourcegraph/sourcegraph/internal/appliance/v1"
 	"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
 	"github.com/sourcegraph/sourcegraph/internal/observation"
@@ -44,7 +49,14 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi
 
 	relregClient := releaseregistry.NewClient(config.relregEndpoint)
 
-	app, err := appliance.NewAppliance(k8sClient, relregClient, config.applianceVersion, config.namespace, logger)
+	noResourceRestrictions := false
+	noResourceRestrictions, err = strconv.ParseBool(config.noResourceRestrictions)
+	if err != nil {
+		logger.Error("parsing APPLIANCE_NO_RESOURCE_RESTRICTIONS as bool", log.Error(err))
+		return err
+	}
+
+	app, err := appliance.NewAppliance(k8sClient, relregClient, config.applianceVersion, config.namespace, noResourceRestrictions, logger)
 	if err != nil {
 		logger.Error("failed to create appliance", log.Error(err))
 		return err
@@ -67,10 +79,13 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi
 		return err
 	}
 
+	beginHealthCheckLoop := make(chan struct{})
+
 	if err = (&reconciler.Reconciler{
-		Client:   mgr.GetClient(),
-		Scheme:   mgr.GetScheme(),
-		Recorder: mgr.GetEventRecorderFor("sourcegraph-appliance"),
+		Client:               mgr.GetClient(),
+		Scheme:               mgr.GetScheme(),
+		Recorder:             mgr.GetEventRecorderFor("sourcegraph-appliance"),
+		BeginHealthCheckLoop: beginHealthCheckLoop,
 	}).SetupWithManager(mgr); err != nil {
 		logger.Error("unable to create the appliance controller", log.Error(err))
 		return err
@@ -92,6 +107,26 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi
 
 	grpcServer := makeGRPCServer(logger, app)
 
+	selfUpdater := &selfupdate.SelfUpdate{
+		Interval:        time.Hour,
+		Logger:          logger.Scoped("SelfUpdate"),
+		K8sClient:       k8sClient,
+		RelregClient:    relregClient,
+		DeploymentNames: config.selfDeploymentName,
+		Namespace:       config.namespace,
+	}
+
+	probe := &healthchecker.PodProbe{K8sClient: k8sClient}
+	healthChecker := &healthchecker.HealthChecker{
+		Probe:     probe,
+		K8sClient: k8sClient,
+		Logger:    logger.Scoped("HealthChecker"),
+
+		ServiceName: types.NamespacedName{Name: "sourcegraph-frontend", Namespace: config.namespace},
+		Interval:    time.Minute,
+		Graceperiod: time.Minute,
+	}
+
 	g, ctx := errgroup.WithContext(ctx)
 	ctx = shutdownOnSignal(ctx)
 
@@ -119,6 +154,18 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi
 		}
 		return nil
 	})
+	g.Go(func() error {
+		if err := healthChecker.ManageIngressFacingService(ctx, beginHealthCheckLoop, "app=sourcegraph-frontend", config.namespace); err != nil {
+			logger.Error("problem running HealthChecker", log.Error(err))
+			return err
+		}
+		return nil
+	})
+	if config.selfDeploymentName != "" {
+		g.Go(func() error {
+			return selfUpdater.Loop(ctx)
+		})
+	}
 	g.Go(func() error {
 		<-ctx.Done()
 		grpcServer.GracefulStop()

diff --git a/cmd/frontend/internal/app/jscontext/jscontext.go b/cmd/frontend/internal/app/jscontext/jscontext.go
@@ -233,6 +233,7 @@ type JSContext struct {
 	CodeIntelRankingDocumentReferenceCountsEnabled bool `json:"codeIntelRankingDocumentReferenceCountsEnabled"`
 
 	CodeInsightsEnabled      bool `json:"codeInsightsEnabled"`
+	ApplianceManaged         bool `json:"applianceManaged"`
 	CodeIntelligenceEnabled  bool `json:"codeIntelligenceEnabled"`
 	SearchContextsEnabled    bool `json:"searchContextsEnabled"`
 	NotebooksEnabled         bool `json:"notebooksEnabled"`
@@ -436,6 +437,7 @@ func NewJSContextFromRequest(req *http.Request, db database.DB) JSContext {
 		CodyRequiresVerifiedEmail: siteResolver.RequiresVerifiedEmailForCody(ctx),
 
 		CodeSearchEnabledOnInstance: codeSearchLicensed,
+		ApplianceManaged:            conf.IsApplianceManaged(),
 
 		ExecutorsEnabled:                               conf.ExecutorsEnabled(),
 		CodeIntelAutoIndexingEnabled:                   conf.CodeIntelAutoIndexingEnabled(),

diff --git a/deps.bzl b/deps.bzl
@@ -6419,13 +6419,6 @@ def go_dependencies():
         sum = "h1:dH55ru2OQOIAKjZi5wwXjNnSfN0oXLFYkMQy908s+tU=",
         version = "v0.2.0",
     )
-    go_repository(
-        name = "com_github_wagslane_go_password_validator",
-        build_file_proto_mode = "disable_global",
-        importpath = "github.com/wagslane/go-password-validator",
-        sum = "h1:vfxOPzGHkz5S146HDpavl0cw1DSVP061Ry2PX0/ON6I=",
-        version = "v0.3.0",
-    )
     go_repository(
         name = "com_github_wk8_go_ordered_map_v2",
         build_file_proto_mode = "disable_global",

diff --git a/dev/BUILD.bazel b/dev/BUILD.bazel
@@ -127,5 +127,6 @@ write_source_files(
         "//cmd/enterprise-portal/internal/subscriptionsservice:generate_mocks",
         "//dev/sg/internal/analytics:generate_mocks",
         "//cmd/symbols/internal/fetcher:generate_mocks",
+        "//internal/releaseregistry/mocks:generate_mocks",
     ],
 )
diff --git a/docker-images/appliance-frontend/BUILD.bazel b/docker-images/appliance-frontend/BUILD.bazel
@@ -0,0 +1,71 @@
+load("//dev:oci_defs.bzl", "image_repository", "oci_image", "oci_push", "oci_tarball")
+load("@rules_pkg//:pkg.bzl", "pkg_tar")
+load("@container_structure_test//:defs.bzl", "container_structure_test")
+load("//wolfi-images:defs.bzl", "wolfi_base")
+
+filegroup(
+    name = "config",
+    srcs = ["nginx.conf"],
+)
+
+filegroup(
+    name = "init_script",
+    srcs = ["init.sh"],
+)
+
+pkg_tar(
+    name = "tar_config",
+    srcs = [":config"],
+    package_dir = "/etc/nginx",
+)
+
+pkg_tar(
+    name = "tar_init_script",
+    srcs = [":init_script"],
+    package_dir = "/",
+)
+
+oci_image(
+    name = "image",
+    base = ":base_image",
+    entrypoint = [
+        "/init.sh",
+        "nginx",
+        "-g",
+        "daemon off;",
+    ],
+    tars = [
+        ":tar_init_script",
+        ":tar_config",
+        "//internal/appliance/frontend/maintenance:tar_config",
+        "//internal/appliance/frontend/maintenance:tar_frontend",
+    ],
+    user = "sourcegraph",
+)
+
+oci_tarball(
+    name = "image_tarball",
+    image = ":image",
+    repo_tags = ["appliance-frontend:candidate"],
+)
+
+container_structure_test(
+    name = "image_test",
+    timeout = "short",
+    configs = ["image_test.yaml"],
+    driver = "docker",
+    image = ":image",
+    tags = [
+        "exclusive",
+        "requires-network",
+        TAG_INFRA_DEVINFRA,
+    ],
+)
+
+oci_push(
+    name = "candidate_push",
+    image = ":image",
+    repository = image_repository("appliance-frontend"),
+)
+
+wolfi_base()