diff --git a/.air.toml b/.air.toml index 6b97b6319..077abe923 100644 --- a/.air.toml +++ b/.air.toml @@ -29,6 +29,7 @@ tmp_dir = "tmp" [log] time = false + main_only = true [misc] clean_on_exit = false diff --git a/charms/jimm-k8s/src/charm.py b/charms/jimm-k8s/src/charm.py index 04c2ee8c7..7480879fd 100755 --- a/charms/jimm-k8s/src/charm.py +++ b/charms/jimm-k8s/src/charm.py @@ -318,7 +318,7 @@ def _update_workload(self, event): if dashboard_relation and self.unit.is_leader(): dashboard_relation.data[self.app].update( { - "controller-url": dns_name, + "controller-url": "wss://{}".format(dns_name), "identity-provider-url": self.config.get("candid-url"), "is-juju": str(False), } @@ -347,7 +347,7 @@ def _on_dashboard_relation_joined(self, event: RelationJoinedEvent): event.relation.data[self.app].update( { - "controller-url": dns_name, + "controller-url": "wss://{}".format(dns_name), "identity-provider-url": self.config["candid-url"], "is-juju": str(False), } diff --git a/charms/jimm-k8s/tests/unit/test_charm.py b/charms/jimm-k8s/tests/unit/test_charm.py index 44950da1e..2652ea7cc 100644 --- a/charms/jimm-k8s/tests/unit/test_charm.py +++ b/charms/jimm-k8s/tests/unit/test_charm.py @@ -269,7 +269,7 @@ def test_dashboard_relation_joined(self): self.assertTrue(data) self.assertEqual( data["controller-url"], - "juju-jimm-k8s-0.juju-jimm-k8s-endpoints.None.svc.cluster.local", + "wss://juju-jimm-k8s-0.juju-jimm-k8s-endpoints.None.svc.cluster.local", ) self.assertEqual(data["identity-provider-url"], "https://candid.example.com") self.assertEqual(data["is-juju"], "False") diff --git a/charms/jimm/README.md b/charms/jimm/README.md index 2ace8e2df..d5b6e248f 100644 --- a/charms/jimm/README.md +++ b/charms/jimm/README.md @@ -43,6 +43,7 @@ juju deploy openfga juju add-relation juju-jimm:openfga postgresql:openfga ``` + ## Developing Create and activate a virtualenv with the development requirements: diff --git a/charms/jimm/charmcraft.yaml b/charms/jimm/charmcraft.yaml index 61ae36388..69e9caa1f 100644 --- a/charms/jimm/charmcraft.yaml +++ b/charms/jimm/charmcraft.yaml @@ -4,16 +4,21 @@ type: "charm" parts: charm: prime: - - ./templates - - ./files - - README.md + - ./templates + - ./files + - README.md charm-python-packages: [setuptools] + charm-binary-python-packages: + - Jinja2 >= 2.11.3 + - markupsafe >= 2.0.1 + - pydantic + - cosl bases: # Ensure run-on is the same or newer than build-on # since jimm-server is a Go binary using CGO dependencies - build-on: - - name: "ubuntu" - channel: "20.04" + - name: "ubuntu" + channel: "20.04" run-on: - - name: "ubuntu" - channel: "20.04" + - name: "ubuntu" + channel: "20.04" diff --git a/charms/jimm/metadata.yaml b/charms/jimm/metadata.yaml index d2f028f21..746bc2b33 100644 --- a/charms/jimm/metadata.yaml +++ b/charms/jimm/metadata.yaml @@ -22,7 +22,7 @@ issues: https://github.com/canonical/jimm/issues description: | JIMM is a juju controller, used in conjunction with the JaaS dashboard to provide a seamless way - to manage models, regardless of where their controllers reside or what cloud they may be running on. + to manage models, regardless of where their controllers reside or what cloud they may be running on. provides: website: diff --git a/charms/jimm/src/charm.py b/charms/jimm/src/charm.py index 09d11d981..05c3bd20b 100755 --- a/charms/jimm/src/charm.py +++ b/charms/jimm/src/charm.py @@ -142,7 +142,6 @@ def _on_config_changed(self, _): with open(self._env_filename(), "wt") as f: f.write(self._render_template("jimm.env", **args)) - if self._ready(): self.restart() self._on_update_status(None) @@ -151,7 +150,7 @@ def _on_config_changed(self, _): if dashboard_relation: dashboard_relation.data[self.app].update( { - "controller-url": self.config["dns-name"], + "controller-url": "wss://{}".format(self.config["dns-name"]), "identity-provider-url": self.config["candid-url"], "is-juju": str(False), } @@ -399,7 +398,7 @@ def _snap(self, *args): def _on_dashboard_relation_joined(self, event): event.relation.data[self.app].update( { - "controller-url": self.config["dns-name"], + "controller-url": "wss://{}".format(self.config["dns-name"]), "identity-provider-url": self.config["candid-url"], "is-juju": str(False), } diff --git a/charms/jimm/tests/test_charm.py b/charms/jimm/tests/test_charm.py index 3598cf129..dc004f5fe 100644 --- a/charms/jimm/tests/test_charm.py +++ b/charms/jimm/tests/test_charm.py @@ -510,7 +510,7 @@ def test_dashboard_relation_joined(self): harness.charm._agent_filename = tmp.name harness.update_config( { - "dns-name": "https://jimm.example.com", + "dns-name": "jimm.example.com", "candid-agent-username": "username@candid", "candid-agent-private-key": "agent-private-key", "candid-agent-public-key": "agent-public-key", @@ -526,7 +526,7 @@ def test_dashboard_relation_joined(self): harness.add_relation_unit(id, "juju-dashboard/0") data = harness.get_relation_data(id, "juju-jimm") self.assertTrue(data) - self.assertEqual(data["controller-url"], "https://jimm.example.com") + self.assertEqual(data["controller-url"], "wss://jimm.example.com") self.assertEqual(data["identity-provider-url"], "https://candid.example.com") self.assertEqual(data["is-juju"], "False") diff --git a/charms/jimm/tox.ini b/charms/jimm/tox.ini index eb08ed4aa..21f4e8632 100644 --- a/charms/jimm/tox.ini +++ b/charms/jimm/tox.ini @@ -9,7 +9,6 @@ envlist = lint, unit [vars] src_path = {toxinidir}/src/ tst_path = {toxinidir}/tests/ -;lib_path = {toxinidir}/lib/charms/operator_name_with_underscores all_path = {[vars]src_path} {[vars]tst_path} [testenv] diff --git a/cmd/jimmctl/cmd/modelstatus.go b/cmd/jimmctl/cmd/modelstatus.go index efaa0f94e..25e741475 100644 --- a/cmd/jimmctl/cmd/modelstatus.go +++ b/cmd/jimmctl/cmd/modelstatus.go @@ -63,7 +63,7 @@ func (c *modelStatusCommand) SetFlags(f *gnuflag.FlagSet) { // Init implements the cmd.Command interface. func (c *modelStatusCommand) Init(args []string) error { - if len(args) < 0 { + if len(args) < 1 { return errors.E("missing model uuid") } c.modelUUID, args = args[0], args[1:] diff --git a/docker-compose.yaml b/docker-compose.yaml index bf7fdaec6..2cd37502a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,6 +33,7 @@ services: ports: - 17070:80 environment: + JIMM_LOG_LEVEL: "debug" JIMM_UUID: "3217dbc9-8ea9-4381-9e97-01eab0b3f6bb" JIMM_DSN: "postgresql://jimm:jimm@db/jimm" CANDID_URL: "http://0.0.0.0:8081" # For external client redirects (in the case of compose and running outside) @@ -78,9 +79,9 @@ services: condition: service_healthy labels: traefik.enable: true - traefik.http.routers.httpd.rule: Host(`jimm.localhost`) - traefik.http.routers.httpd.entrypoints: websecure - traefik.http.routers.httpd.tls: true + traefik.http.routers.jimm.rule: Host(`jimm.localhost`) + traefik.http.routers.jimm.entrypoints: websecure + traefik.http.routers.jimm.tls: true db: image: postgres @@ -134,8 +135,6 @@ services: image: candid:latest container_name: candid entrypoint: "/candid.sh" - command: "" - # command: "/etc/candid/config.yaml && echo 'hi' && ls" expose: - 8081 ports: diff --git a/internal/dashboard/dashboard.go b/internal/dashboard/dashboard.go index 9e64f7c1c..d2e1ddc93 100644 --- a/internal/dashboard/dashboard.go +++ b/internal/dashboard/dashboard.go @@ -118,7 +118,6 @@ func Handler(ctx context.Context, loc string) http.Handler { ) continue } - type guiArchiveVersion struct { // Version holds the Juju GUI version number. Version version.Number `json:"version"` diff --git a/internal/dashboard/dashboard_test.go b/internal/dashboard/dashboard_test.go index 97b16924b..8bb3283b0 100644 --- a/internal/dashboard/dashboard_test.go +++ b/internal/dashboard/dashboard_test.go @@ -183,7 +183,6 @@ func TestGUIArchiveEndpoint(t *testing.T) { c.Check(resp.StatusCode, qt.Equals, http.StatusOK) buf, err := io.ReadAll(resp.Body) c.Assert(err, qt.IsNil) - c.Check( string(buf), qt.Equals, diff --git a/internal/jujuapi/applicationoffers.go b/internal/jujuapi/applicationoffers.go index 67f7afa0b..5d55d094c 100644 --- a/internal/jujuapi/applicationoffers.go +++ b/internal/jujuapi/applicationoffers.go @@ -60,7 +60,7 @@ func init() { r.AddMethod("ApplicationOffers", 4, "FindApplicationOffers", findOffersMethod) r.AddMethod("ApplicationOffers", 4, "ApplicationOffers", applicationOffersMethod) - return []int{1, 2, 3} + return []int{1, 2, 3, 4} } } diff --git a/internal/jujuapi/jimm.go b/internal/jujuapi/jimm.go index c21cfac10..9f4ee1f6c 100644 --- a/internal/jujuapi/jimm.go +++ b/internal/jujuapi/jimm.go @@ -63,6 +63,7 @@ func init() { r.AddMethod("JIMM", 3, "UpdateMigratedModel", updateMigratedModelMethod) r.AddMethod("JIMM", 3, "AddCloudToController", addCloudToControllerMethod) r.AddMethod("JIMM", 3, "RemoveCloudFromController", removeCloudFromControllerMethod) + r.AddMethod("JIMM", 3, "CrossModelQuery", crossModelQueryMethod) // JIMM Generic RPC r.AddMethod("JIMM", 4, "AddController", addControllerMethod) diff --git a/internal/jujuclient/modelmanager_test.go b/internal/jujuclient/modelmanager_test.go index b74926eae..610bf51a5 100644 --- a/internal/jujuclient/modelmanager_test.go +++ b/internal/jujuclient/modelmanager_test.go @@ -183,7 +183,6 @@ func (s *modelmanagerSuite) TestRevokeModelAccessError(c *gc.C) { func (s *modelmanagerSuite) TestValidateModelUpgrade(c *gc.C) { c.Skip("juju 3.x no longer implements this method") - ctx := context.Background() args := jujuparams.ModelCreateArgs{ diff --git a/local/README.md b/local/README.md index 8c5df3431..21614b9b1 100644 --- a/local/README.md +++ b/local/README.md @@ -10,8 +10,8 @@ used for integration testing within the JIMM test suite. 3. Run `make pull/candid` to get a local image of candid (this is subject to change!) 4. Run `cd local/traefik/certs; ./certs.sh; cd -`, this will setup some self signed certs and add them to your cert pool. 5. Run `touch ./local/vault/approle.json && touch ./local/vault/roleid.txt` -6. Run `make version/commit.txt` to populate the repo with the git commit info. -7. Run `make version/version.txt` to populate the repo with the git version info. +6. Run `make version/commit.txt && make version/version.txt` to populate the repo with the git commit and version info. +7. Run `go mod vendor` to vendor JIMM's dependencies and reduce repeated setup time. 8. `docker compose --profile dev up` if you encounter an error like "Error response from daemon: network ... not found" then the command `docker compose --profile dev up --force-recreate` should help. After this initial setup, subsequent use of the compose can be done with `docker compose --profile dev up --force-recreate` @@ -27,17 +27,6 @@ The services included are: to re-run the compose continuously, but note, if you do bring the compose down, remove the volumes otherwise vault will not behave correctly, this can be done via `docker compose down -v` -If all was successful, you should seen an output similar to: -``` -NAME COMMAND SERVICE STATUS PORTS -candid candid:latest "/candid.sh" candid About a minute ago Up About a minute (healthy) ... -jimmy cosmtrek/air "/go/bin/air" jimm About a minute ago Up 54 seconds (healthy) ... -openfga jimm-openfga "/app/openfga run" openfga About a minute ago Up About a minute (healthy) ... -postgres postgres "docker-entrypoint.s…" db About a minute ago Up About a minute (healthy) ... -traefik traefik:2.9 "/entrypoint.sh trae…" traefik About a minute ago Up About a minute (healthy) ... -vault vault:latest "docker-entrypoint.s…" vault About a minute ago Up About a minute (unhealthy) ... -``` - Now please checkout the [Authentication Steps](#authentication-steps) to authenticate postman for local testing & Q/A. # Q/A Using Postman @@ -56,48 +45,18 @@ The `request name` represents the literal WS endpoint, i.e., `API = /api`. # Q/A Using jimmctl -## Prerequisited +## Prerequisites -1. Make sure you are using latest juju from their develop branch (at the moment, we're using 3.2-beta1). -2. change jujuClientVersion in `internal/jujuclient/dial.go` to **3.2-beta1** by running: ``sed -i 's/jujuClientVersion = "2.9.42"/jujuClientVersion = "3.2-beta1"/g' ./internal/jujuclient/dial.go`` +// TODO(): Ipv6 network on the Juju container don't work with JIMM. Figure out how to disable these at the container level so that the controller.yaml file doesn't present ipv6 at all. For now one can remove this by hand. -Steps: +Note that you can export an environment variable `CONTROLLER_NAME` and re-run steps 3. and 4. below to create multiple Juju +controllers that will be controlled by JIMM. -Manual: 1. `juju unregister jimm-dev` - Unregister any other local JIMM you have. -2. `juju login jimm.localhost -c jimm-dev` - Login to local JIMM. You don't have to do this, you will be logged in automatically. -3. `juju bootstrap localhost qa-controller --config allow-model-access=true --config login-token-refresh-url=https://jimm.localhost` - Bootstrap a Q/A controller. - 1. run `lxc list` to name of the machine running the juju controller, this will resemble "juju-d5ede3-0" - 2. export name of the machine using `export TEST_JUJU_CTRL=` - 3. create a proxy on the controller machine that will enable controller to reach jimm by running `lxc config device add "${TEST_JUJU_CTRL}" myproxy proxy listen=tcp:0.0.0.0:443 connect=tcp:127.0.0.1:443 bind=instance` - 4. we also need to push the CA cert that was used to create JIMM's certificate to the machine by running `lxc file push local/traefik/certs/ca.crt "${TEST_JUJU_CTRL}"/usr/local/share/ca-certificates/` - 5. now go into the controller `lxc shell "${TEST_JUJU_CTRL}"` - 6. and run `update-ca-certificates` to update machine's CA certs - 7. we also want to add an entry to `/etc/hosts` that will allow the controller to correctly resolve `jimm.localhost` and use the proxy we created in step 3. Run `echo "127.0.0.1 jimm.localhost" >> /etc/hosts` - 8. next we need to start and stop the controller by running `lxc stop "${TEST_JUJU_CTRL}"` and then `lxc start "${TEST_JUJU_CTRL}"` because controller fetches the jwks from jimm on startup at which point the proxy had not yet been set up. -4. `go build ./cmd/jimmctl` -5. `./jimmctl controller-info qa-controller ./qa-controller.yaml` - Get Q/A controller info. - 1. Modify qa-controller.yaml public address to "juju-apiserver:17070" - 2. Run `docker compose exec -it jimm bash` and update the /etc/hosts to have the api-addresses point to "juju-apiserver" - Build CLI tool. -6. `juju switch jimm-dev` - Switch back to JIMM controller. -7. `./jimmctl add-controller ./qa-controller.yaml` - Add Q/A controller to JIMM. -8. `juju update-credentials localhost --controller jimm-dev` - Add client credentials for qa-controller's cloud (localhost) to JIMM's controller credential list. -9. `juju add-model test` - Adds a model to qa-controller via JIMM. - -Semi-automated: -0. `juju unregister jimm-dev` - Unregister any other local JIMM you have. -1. `juju login jimm.localhost -c jimm-dev` - Login to local JIMM. (If you name the controller jimm-dev, the script will pick it up!) -2. `juju bootstrap localhost qa-controller --config allow-model-access=true --config login-token-refresh-url=https://jimm.localhost` - - Bootstrap a Q/A controller. (If you name the controller qa-controller and the cloud is lxd or microk8s, the script will pick it up!) - 1. run `lxc list` to name of the machine running the juju controller, this will resemble "juju-d5ede3-0" - 2. export name of the machine using `export TEST_JUJU_CTRL=` - 3. create a proxy on the controller machine that will enable controller to reach jimm by running `lxc config device add "${TEST_JUJU_CTRL}" myproxy proxy listen=tcp:0.0.0.0:443 connect=tcp:127.0.0.1:443 bind=instance` - 4. we also need to push the CA cert that was used to create JIMM's certificate to the machine by running `lxc file push local/traefik/certs/ca.crt "${TEST_JUJU_CTRL}"/usr/local/share/ca-certificates/` - 5. now go into the controller `lxc shell "${TEST_JUJU_CTRL}"` - 6. and run `update-ca-certificates` to update machine's CA certs - 7. we also want to add an entry to `/etc/hosts` that will allow the controller to correctly resolve `jimm.localhost` and use the proxy we created in step 3. Run `echo "127.0.0.1 jimm.localhost" >> /etc/hosts` - 8. next we need to start and stop the controller by running `lxc stop "${TEST_JUJU_CTRL}"` and then `lxc start "${TEST_JUJU_CTRL}"` because controller fetches the jwks from jimm on startup at which point the proxy had not yet been set up. -3. `./local/jimm/add-controller.sh` - A local script to do many of the manual steps for us. See script for more details. -4. `juju add-model test` - Adds a model to qa-controller via JIMM. +2. `juju login jimm.localhost -c jimm-dev` - Login to local JIMM. (If you name the controller jimm-dev, the script will pick it up!) +3. `./local/jimm/setup-controller.sh` - Performs controller setup. +4. `./local/jimm/add-controller.sh` - A local script to do many of the manual steps for us. See script for more details. +5. `juju add-model test` - Adds a model to qa-controller via JIMM. # Helpful tidbits! > Note: For any secure step to work, ensure you've run the local traefik certs script! @@ -105,4 +64,4 @@ Semi-automated: - To access vault UI, the URL is: `http://localhost:8200/ui` and the root key is `token`. - The WS API for JIMM Controller is under: `ws://localhost:17070` (http direct) and `wss://jimm.localhost` for secure. - You can verify local deployment with: `curl http://localhost:17070/debug/status` and `curl https://jimm.localhost/debug/status` -- Traefik is available on `http://localhost:8089`. \ No newline at end of file +- Traefik is available on `http://localhost:8089`. diff --git a/local/candid/entry.sh b/local/candid/entry.sh index 82cb8d3c0..3ddeac211 100755 --- a/local/candid/entry.sh +++ b/local/candid/entry.sh @@ -3,16 +3,6 @@ echo "Entrypoint being overriden for local environment." -# Grab curl quickly. apt update apt install curl -y -/root/candidsrv /etc/candid/config.yaml & - -# Pseudo readiness probe such that we can continue local dev setup. -until eval curl --output /dev/null --silent --fail http://localhost:8081/debug/status; do - printf '.' - sleep 1 -done -echo "Server appears to have started." -# If any further configuration to the IdP is required, it can now be done via this script. -wait \ No newline at end of file +exec /root/candidsrv /etc/candid/config.yaml diff --git a/local/jimm/add-controller.sh b/local/jimm/add-controller.sh index ccade5728..f008c7254 100755 --- a/local/jimm/add-controller.sh +++ b/local/jimm/add-controller.sh @@ -10,41 +10,17 @@ # # Requirements to run this script: # - yq (snap) +set -eux -JIMM_CONTROLLER_NAME=$1 -CONTROLLER_NAME=$2 -CONTROLLER_YAML_PATH=$3 -CLIENT_CREDENTIAL_NAME=$4 - -echo "Checking environment..." -if [ -z "$JIMM_CONTROLLER_NAME" ]; -then - echo "- JIMM controller name NOT SET, setting it to 'jimm-dev'" - JIMM_CONTROLLER_NAME="jimm-dev" -fi - -if [ -z "$CONTROLLER_NAME" ]; -then - echo "- Controller name NOT SET, setting it to 'qa-controller'" - CONTROLLER_NAME="qa-controller" -fi - -if [ -z "$CONTROLLER_YAML_PATH" ]; -then - echo "- Controller YAML path NOT SET, setting it to './qa-controller'" - CONTROLLER_YAML_PATH="./qa-controller" -fi - -if [ -z "$CLIENT_CREDENTIAL_NAME" ]; -then - echo "- Client credential name NOT SET, setting it to 'microk8s'" - CLIENT_CREDENTIAL_NAME="localhost" -fi +JIMM_CONTROLLER_NAME="${JIMM_CONTROLLER_NAME:-jimm-dev}" +CONTROLLER_NAME="${CONTROLLER_NAME:-qa-controller}" +CONTROLLER_YAML_PATH="${CONTROLLER_NAME}".yaml +CLIENT_CREDENTIAL_NAME="${CLIENT_CREDENTIAL_NAME:-localhost}" echo echo "JIMM controller name is: $JIMM_CONTROLLER_NAME" -echo "Targ controller name is: $CONTROLLER_NAME" -echo "Targ controller path is: $CONTROLLER_YAML_PATH" +echo "Target controller name is: $CONTROLLER_NAME" +echo "Target controller path is: $CONTROLLER_YAML_PATH" echo echo "Building jimmctl..." # Build jimmctl so we may add a controller. @@ -63,16 +39,6 @@ else exit 1 fi echo -echo "Retrieving controller address" -CONTROLLER_ADDRESS=$(cat "$CONTROLLER_YAML_PATH" | yq '.public-address' | cut -d ":" -f 1) -echo "Controller address is: $CONTROLLER_ADDRESS" -echo -echo "Updating $CONTROLLER_YAML_PATH public-address..." -yq -i e '.public-address |= "juju-apiserver:17070"' "$CONTROLLER_YAML_PATH" -echo -echo "Updating containers /etc/hosts..." -docker compose exec -w /etc --no-TTY jimm bash -c "echo '$CONTROLLER_ADDRESS juju-apiserver' >> hosts" -echo echo "Adding controller from path: $CONTROLLER_YAML_PATH" ./jimmctl add-controller "$CONTROLLER_YAML_PATH" echo diff --git a/local/jimm/setup-controller.sh b/local/jimm/setup-controller.sh new file mode 100755 index 000000000..155279ee2 --- /dev/null +++ b/local/jimm/setup-controller.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# RUN THIS SCRIPT FROM PROJECT ROOT! +# It will bootstrap a Juju controller and configure the necessary config to enable the controller +# to communicate with the docker compose + +set -ux + +CONTROLLER_NAME="${CONTROLLER_NAME:-qa-controller}" + +echo "Bootstrapping controller" +juju bootstrap localhost "${CONTROLLER_NAME}" --config allow-model-access=true --config login-token-refresh-url=https://jimm.localhost +CONTROLLER_ID=$(juju show-controller --format json | jq --arg name "${CONTROLLER_NAME}" '.[$name]."controller-machines"."0"."instance-id"' | tr -d '"') +echo "Adding proxy to LXC instance ${CONTROLLER_ID}" +lxc config device add "${CONTROLLER_ID}" myproxy proxy listen=tcp:0.0.0.0:443 connect=tcp:127.0.0.1:443 bind=instance +echo "Pushing local CA" +lxc file push local/traefik/certs/ca.crt "${CONTROLLER_ID}"/usr/local/share/ca-certificates/ +lxc exec "${CONTROLLER_ID}" -- update-ca-certificates +echo "Restarting controller" +lxc stop "${CONTROLLER_ID}" +lxc start "${CONTROLLER_ID}" diff --git a/service.go b/service.go index 6d71c59e3..6d027bfe9 100644 --- a/service.go +++ b/service.go @@ -465,7 +465,7 @@ func newVaultStore(ctx context.Context, p Params) (jimmcreds.CredentialStore, er } defer f.Close() s, err := vaultapi.ParseSecret(f) - if err != nil { + if err != nil || s == nil { zapctx.Error(ctx, "failed to parse vault secret from file") return nil, err }