Merged main and removed conflicts

.air.toml

@@ -29,6 +29,7 @@ tmp_dir = "tmp"
 
 [log]
   time = false
+  main_only = true
 
 [misc]
   clean_on_exit = false

charms/jimm-k8s/src/charm.py

@@ -318,7 +318,7 @@ def _update_workload(self, event):
         if dashboard_relation and self.unit.is_leader():
             dashboard_relation.data[self.app].update(
                 {
-                    "controller-url": dns_name,
+                    "controller-url": "wss://{}".format(dns_name),
                     "identity-provider-url": self.config.get("candid-url"),
                     "is-juju": str(False),
                 }
@@ -347,7 +347,7 @@ def _on_dashboard_relation_joined(self, event: RelationJoinedEvent):
 
         event.relation.data[self.app].update(
             {
-                "controller-url": dns_name,
+                "controller-url": "wss://{}".format(dns_name),
                 "identity-provider-url": self.config["candid-url"],
                 "is-juju": str(False),
             }

charms/jimm-k8s/tests/unit/test_charm.py

@@ -269,7 +269,7 @@ def test_dashboard_relation_joined(self):
         self.assertTrue(data)
         self.assertEqual(
             data["controller-url"],
-            "juju-jimm-k8s-0.juju-jimm-k8s-endpoints.None.svc.cluster.local",
+            "wss://juju-jimm-k8s-0.juju-jimm-k8s-endpoints.None.svc.cluster.local",
         )
         self.assertEqual(data["identity-provider-url"], "https://candid.example.com")
         self.assertEqual(data["is-juju"], "False")

charms/jimm/README.md

@@ -43,6 +43,7 @@ juju deploy openfga
 juju add-relation juju-jimm:openfga postgresql:openfga
 ```
 
+
 ## Developing
 
 Create and activate a virtualenv with the development requirements:

charms/jimm/charmcraft.yaml

@@ -4,16 +4,21 @@ type: "charm"
 parts:
   charm:
     prime:
-    - ./templates
-    - ./files
-    - README.md
+      - ./templates
+      - ./files
+      - README.md
     charm-python-packages: [setuptools]
+    charm-binary-python-packages:
+      - Jinja2 >= 2.11.3
+      - markupsafe >= 2.0.1
+      - pydantic
+      - cosl
 bases:
   # Ensure run-on is the same or newer than build-on
   # since jimm-server is a Go binary using CGO dependencies
   - build-on:
-    - name: "ubuntu"
-      channel: "20.04"
+      - name: "ubuntu"
+        channel: "20.04"
     run-on:
-    - name: "ubuntu"
-      channel: "20.04"
+      - name: "ubuntu"
+        channel: "20.04"

charms/jimm/metadata.yaml

@@ -22,7 +22,7 @@ issues: https://github.com/canonical/jimm/issues
 
 description: |
   JIMM is a juju controller, used in conjunction with the JaaS dashboard to provide a seamless way
-  to manage models, regardless of where their controllers reside or what cloud they may be running on. 
+  to manage models, regardless of where their controllers reside or what cloud they may be running on.
 
 provides:
   website:

charms/jimm/src/charm.py

@@ -142,7 +142,6 @@ def _on_config_changed(self, _):
 
         with open(self._env_filename(), "wt") as f:
             f.write(self._render_template("jimm.env", **args))
-
         if self._ready():
             self.restart()
         self._on_update_status(None)
@@ -151,7 +150,7 @@ def _on_config_changed(self, _):
         if dashboard_relation:
             dashboard_relation.data[self.app].update(
                 {
-                    "controller-url": self.config["dns-name"],
+                    "controller-url": "wss://{}".format(self.config["dns-name"]),
                     "identity-provider-url": self.config["candid-url"],
                     "is-juju": str(False),
                 }
@@ -399,7 +398,7 @@ def _snap(self, *args):
     def _on_dashboard_relation_joined(self, event):
         event.relation.data[self.app].update(
             {
-                "controller-url": self.config["dns-name"],
+                "controller-url": "wss://{}".format(self.config["dns-name"]),
                 "identity-provider-url": self.config["candid-url"],
                 "is-juju": str(False),
             }

charms/jimm/tests/test_charm.py

@@ -510,7 +510,7 @@ def test_dashboard_relation_joined(self):
             harness.charm._agent_filename = tmp.name
             harness.update_config(
                 {
-                    "dns-name": "https://jimm.example.com",
+                    "dns-name": "jimm.example.com",
                     "candid-agent-username": "username@candid",
                     "candid-agent-private-key": "agent-private-key",
                     "candid-agent-public-key": "agent-public-key",
@@ -526,7 +526,7 @@ def test_dashboard_relation_joined(self):
         harness.add_relation_unit(id, "juju-dashboard/0")
         data = harness.get_relation_data(id, "juju-jimm")
         self.assertTrue(data)
-        self.assertEqual(data["controller-url"], "https://jimm.example.com")
+        self.assertEqual(data["controller-url"], "wss://jimm.example.com")
         self.assertEqual(data["identity-provider-url"], "https://candid.example.com")
         self.assertEqual(data["is-juju"], "False")
 

charms/jimm/tox.ini

@@ -9,7 +9,6 @@ envlist = lint, unit
 [vars]
 src_path = {toxinidir}/src/
 tst_path = {toxinidir}/tests/
-;lib_path = {toxinidir}/lib/charms/operator_name_with_underscores
 all_path = {[vars]src_path} {[vars]tst_path} 
 
 [testenv]

cmd/jimmctl/cmd/modelstatus.go

@@ -63,7 +63,7 @@ func (c *modelStatusCommand) SetFlags(f *gnuflag.FlagSet) {
 
 // Init implements the cmd.Command interface.
 func (c *modelStatusCommand) Init(args []string) error {
-	if len(args) < 0 {
+	if len(args) < 1 {
 		return errors.E("missing model uuid")
 	}
 	c.modelUUID, args = args[0], args[1:]

docker-compose.yaml

@@ -33,6 +33,7 @@ services:
     ports:
       - 17070:80
     environment:
+      JIMM_LOG_LEVEL: "debug"
       JIMM_UUID: "3217dbc9-8ea9-4381-9e97-01eab0b3f6bb"
       JIMM_DSN: "postgresql://jimm:jimm@db/jimm"
       CANDID_URL: "http://0.0.0.0:8081" # For external client redirects (in the case of compose and running outside)
@@ -78,9 +79,9 @@ services:
         condition: service_healthy
     labels:
       traefik.enable: true
-      traefik.http.routers.httpd.rule: Host(`jimm.localhost`)
-      traefik.http.routers.httpd.entrypoints: websecure
-      traefik.http.routers.httpd.tls: true
+      traefik.http.routers.jimm.rule: Host(`jimm.localhost`)
+      traefik.http.routers.jimm.entrypoints: websecure
+      traefik.http.routers.jimm.tls: true
 
   db:
     image: postgres
@@ -134,8 +135,6 @@ services:
     image: candid:latest
     container_name: candid
     entrypoint: "/candid.sh"
-    command: ""
-    # command: "/etc/candid/config.yaml && echo 'hi' && ls"
     expose:
       - 8081
     ports:

internal/dashboard/dashboard.go

@@ -118,7 +118,6 @@ func Handler(ctx context.Context, loc string) http.Handler {
 				)
 				continue
 			}
-
 			type guiArchiveVersion struct {
 				// Version holds the Juju GUI version number.
 				Version version.Number `json:"version"`

internal/dashboard/dashboard_test.go

@@ -183,7 +183,6 @@ func TestGUIArchiveEndpoint(t *testing.T) {
 	c.Check(resp.StatusCode, qt.Equals, http.StatusOK)
 	buf, err := io.ReadAll(resp.Body)
 	c.Assert(err, qt.IsNil)
-
 	c.Check(
 		string(buf),
 		qt.Equals,

internal/jujuapi/applicationoffers.go

@@ -60,7 +60,7 @@ func init() {
 		r.AddMethod("ApplicationOffers", 4, "FindApplicationOffers", findOffersMethod)
 		r.AddMethod("ApplicationOffers", 4, "ApplicationOffers", applicationOffersMethod)
 
-		return []int{1, 2, 3}
+		return []int{1, 2, 3, 4}
 	}
 }
 

internal/jujuapi/jimm.go

@@ -63,6 +63,7 @@ func init() {
 		r.AddMethod("JIMM", 3, "UpdateMigratedModel", updateMigratedModelMethod)
 		r.AddMethod("JIMM", 3, "AddCloudToController", addCloudToControllerMethod)
 		r.AddMethod("JIMM", 3, "RemoveCloudFromController", removeCloudFromControllerMethod)
+		r.AddMethod("JIMM", 3, "CrossModelQuery", crossModelQueryMethod)
 
 		// JIMM Generic RPC
 		r.AddMethod("JIMM", 4, "AddController", addControllerMethod)

internal/jujuclient/modelmanager_test.go

@@ -183,7 +183,6 @@ func (s *modelmanagerSuite) TestRevokeModelAccessError(c *gc.C) {
 
 func (s *modelmanagerSuite) TestValidateModelUpgrade(c *gc.C) {
 	c.Skip("juju 3.x no longer implements this method")
-
 	ctx := context.Background()
 
 	args := jujuparams.ModelCreateArgs{

local/README.md

@@ -10,8 +10,8 @@ used for integration testing within the JIMM test suite.
 3. Run `make pull/candid` to get a local image of candid (this is subject to change!)
 4. Run `cd local/traefik/certs; ./certs.sh; cd -`, this will setup some self signed certs and add them to your cert pool.
 5. Run `touch ./local/vault/approle.json && touch ./local/vault/roleid.txt`
-6. Run `make version/commit.txt` to populate the repo with the git commit info.
-7. Run `make version/version.txt` to populate the repo with the git version info.
+6. Run `make version/commit.txt && make version/version.txt` to populate the repo with the git commit and version info.
+7. Run `go mod vendor` to vendor JIMM's dependencies and reduce repeated setup time.
 8. `docker compose --profile dev up` if you encounter an error like "Error response from daemon: network ... not found" then the command `docker compose --profile dev up --force-recreate` should help.
 
 After this initial setup, subsequent use of the compose can be done with `docker compose --profile dev up --force-recreate`
@@ -27,17 +27,6 @@ The services included are:
 to re-run the compose continuously, but note, if you do bring the compose down, remove the volumes otherwise
 vault will not behave correctly, this can be done via `docker compose down -v`
 
-If all was successful, you should seen an output similar to:
-```
-NAME                COMMAND                  SERVICE             STATUS                PORTS
-candid              candid:latest       "/candid.sh"             candid              About a minute ago   Up About a minute (healthy)     ...
-jimmy               cosmtrek/air        "/go/bin/air"            jimm                About a minute ago   Up 54 seconds (healthy)         ...
-openfga             jimm-openfga        "/app/openfga run"       openfga             About a minute ago   Up About a minute (healthy)     ...
-postgres            postgres            "docker-entrypoint.s…"   db                  About a minute ago   Up About a minute (healthy)     ...
-traefik             traefik:2.9         "/entrypoint.sh trae…"   traefik             About a minute ago   Up About a minute (healthy)     ...
-vault               vault:latest        "docker-entrypoint.s…"   vault               About a minute ago   Up About a minute (unhealthy)   ...
-```
-
 Now please checkout the [Authentication Steps](#authentication-steps) to authenticate postman for local testing & Q/A.
 
 # Q/A Using Postman
@@ -56,53 +45,23 @@ The `request name` represents the literal WS endpoint, i.e., `API = /api`.
 
 # Q/A Using jimmctl
 
-## Prerequisited
+## Prerequisites
 
-1. Make sure you are using latest juju from their develop branch (at the moment, we're using 3.2-beta1).
-2. change jujuClientVersion in `internal/jujuclient/dial.go` to **3.2-beta1** by running: ``sed -i 's/jujuClientVersion = "2.9.42"/jujuClientVersion = "3.2-beta1"/g' ./internal/jujuclient/dial.go``
+// TODO(): Ipv6 network on the Juju container don't work with JIMM. Figure out how to disable these at the container level so that the controller.yaml file doesn't present ipv6 at all. For now one can remove this by hand.
 
-Steps:
+Note that you can export an environment variable `CONTROLLER_NAME` and re-run steps 3. and 4. below to create multiple Juju
+controllers that will be controlled by JIMM.
 
-Manual:
 1. `juju unregister jimm-dev`                                       - Unregister any other local JIMM you have.
-2. `juju login jimm.localhost -c jimm-dev`                          - Login to local JIMM. You don't have to do this, you will be logged in automatically.
-3. `juju bootstrap localhost qa-controller --config allow-model-access=true --config login-token-refresh-url=https://jimm.localhost`                          - Bootstrap a Q/A controller.
-   1. run `lxc list` to name of the machine running the juju controller, this will resemble "juju-d5ede3-0"
-   2. export name of the machine using `export TEST_JUJU_CTRL=<instance-name>`
-   3. create a proxy on the controller machine that will enable controller to reach jimm by running `lxc config device add "${TEST_JUJU_CTRL}" myproxy proxy listen=tcp:0.0.0.0:443 connect=tcp:127.0.0.1:443 bind=instance`
-   4. we also need to push the CA cert that was used to create JIMM's certificate to the machine by running `lxc file push local/traefik/certs/ca.crt "${TEST_JUJU_CTRL}"/usr/local/share/ca-certificates/`
-   5. now go into the controller `lxc shell "${TEST_JUJU_CTRL}"`
-   6. and run `update-ca-certificates` to update machine's CA certs
-   7. we also want to add an entry to `/etc/hosts` that will allow the controller to correctly resolve `jimm.localhost` and use the proxy we created in step 3. Run `echo "127.0.0.1 jimm.localhost" >> /etc/hosts`
-   8. next we need to start and stop the controller by running `lxc stop "${TEST_JUJU_CTRL}"` and then `lxc start "${TEST_JUJU_CTRL}"` because controller fetches the jwks from jimm on startup at which point the proxy had not yet been set up.
-4. `go build ./cmd/jimmctl`     
-5. `./jimmctl controller-info qa-controller ./qa-controller.yaml`   - Get Q/A controller info.
-   1. Modify qa-controller.yaml public address to "juju-apiserver:17070"
-   2. Run `docker compose exec -it jimm bash` and update the /etc/hosts to have the api-addresses point to "juju-apiserver"                                    - Build CLI tool.
-6. `juju switch jimm-dev`                                           - Switch back to JIMM controller.
-7. `./jimmctl add-controller ./qa-controller.yaml`                  - Add Q/A controller to JIMM.
-8. `juju update-credentials localhost --controller jimm-dev`         - Add client credentials for qa-controller's cloud (localhost) to JIMM's controller credential list. 
-9. `juju add-model test`                                            - Adds a model to qa-controller via JIMM.
-
-Semi-automated:
-0. `juju unregister jimm-dev`                                       - Unregister any other local JIMM you have.
-1. `juju login jimm.localhost -c jimm-dev`                          - Login to local JIMM. (If you name the controller jimm-dev, the script will pick it up!)
-2. `juju bootstrap localhost qa-controller --config allow-model-access=true --config login-token-refresh-url=https://jimm.localhost`                          - - Bootstrap a Q/A controller. (If you name the controller qa-controller and the cloud is lxd or microk8s, the script will pick it up!)
-   1. run `lxc list` to name of the machine running the juju controller, this will resemble "juju-d5ede3-0"
-   2. export name of the machine using `export TEST_JUJU_CTRL=<instance-name>`
-   3. create a proxy on the controller machine that will enable controller to reach jimm by running `lxc config device add "${TEST_JUJU_CTRL}" myproxy proxy listen=tcp:0.0.0.0:443 connect=tcp:127.0.0.1:443 bind=instance`
-   4. we also need to push the CA cert that was used to create JIMM's certificate to the machine by running `lxc file push local/traefik/certs/ca.crt "${TEST_JUJU_CTRL}"/usr/local/share/ca-certificates/`
-   5. now go into the controller `lxc shell "${TEST_JUJU_CTRL}"`
-   6. and run `update-ca-certificates` to update machine's CA certs
-   7. we also want to add an entry to `/etc/hosts` that will allow the controller to correctly resolve `jimm.localhost` and use the proxy we created in step 3. Run `echo "127.0.0.1 jimm.localhost" >> /etc/hosts`
-   8. next we need to start and stop the controller by running `lxc stop "${TEST_JUJU_CTRL}"` and then `lxc start "${TEST_JUJU_CTRL}"` because controller fetches the jwks from jimm on startup at which point the proxy had not yet been set up.
-3. `./local/jimm/add-controller.sh`                                 - A local script to do many of the manual steps for us. See script for more details.
-4. `juju add-model test`                                            - Adds a model to qa-controller via JIMM.
+2. `juju login jimm.localhost -c jimm-dev`                          - Login to local JIMM. (If you name the controller jimm-dev, the script will pick it up!)
+3. `./local/jimm/setup-controller.sh`                               - Performs controller setup.
+4. `./local/jimm/add-controller.sh`                                 - A local script to do many of the manual steps for us. See script for more details.
+5. `juju add-model test`                                            - Adds a model to qa-controller via JIMM.
 
 # Helpful tidbits!
 > Note: For any secure step to work, ensure you've run the local traefik certs script!
 
 - To access vault UI, the URL is: `http://localhost:8200/ui` and the root key is `token`.
 - The WS API for JIMM Controller is under: `ws://localhost:17070` (http direct) and `wss://jimm.localhost` for secure.
 - You can verify local deployment with: `curl http://localhost:17070/debug/status` and `curl https://jimm.localhost/debug/status`
-- Traefik is available on `http://localhost:8089`.
+- Traefik is available on `http://localhost:8089`.

local/candid/entry.sh

@@ -3,16 +3,6 @@
 
 echo "Entrypoint being overriden for local environment."
 
-# Grab curl quickly.
 apt update
 apt install curl -y
-/root/candidsrv /etc/candid/config.yaml &
-
-# Pseudo readiness probe such that we can continue local dev setup.
-until eval curl --output /dev/null --silent --fail http://localhost:8081/debug/status; do
-    printf '.'
-    sleep 1
-done
-echo "Server appears to have started."
-# If any further configuration to the IdP is required, it can now be done via this script.
-wait
+exec /root/candidsrv /etc/candid/config.yaml

local/jimm/add-controller.sh

@@ -10,41 +10,17 @@
 #
 # Requirements to run this script:
 # - yq (snap)
+set -eux
 
-JIMM_CONTROLLER_NAME=$1
-CONTROLLER_NAME=$2
-CONTROLLER_YAML_PATH=$3
-CLIENT_CREDENTIAL_NAME=$4
-
-echo "Checking environment..."
-if [ -z "$JIMM_CONTROLLER_NAME" ];
-then
-    echo "- JIMM controller name NOT SET, setting it to 'jimm-dev'"
-    JIMM_CONTROLLER_NAME="jimm-dev"
-fi
-
-if [ -z "$CONTROLLER_NAME" ];
-then
-    echo "- Controller name NOT SET, setting it to 'qa-controller'"
-    CONTROLLER_NAME="qa-controller"
-fi
-
-if [ -z "$CONTROLLER_YAML_PATH" ];
-then
-    echo "- Controller YAML path NOT SET, setting it to './qa-controller'"
-    CONTROLLER_YAML_PATH="./qa-controller"
-fi
-
-if [ -z "$CLIENT_CREDENTIAL_NAME" ];
-then
-    echo "- Client credential name NOT SET, setting it to 'microk8s'"
-    CLIENT_CREDENTIAL_NAME="localhost"
-fi
+JIMM_CONTROLLER_NAME="${JIMM_CONTROLLER_NAME:-jimm-dev}"
+CONTROLLER_NAME="${CONTROLLER_NAME:-qa-controller}"
+CONTROLLER_YAML_PATH="${CONTROLLER_NAME}".yaml
+CLIENT_CREDENTIAL_NAME="${CLIENT_CREDENTIAL_NAME:-localhost}"
 
 echo
 echo "JIMM controller name is: $JIMM_CONTROLLER_NAME"
-echo "Targ controller name is: $CONTROLLER_NAME"
-echo "Targ controller path is: $CONTROLLER_YAML_PATH"
+echo "Target controller name is: $CONTROLLER_NAME"
+echo "Target controller path is: $CONTROLLER_YAML_PATH"
 echo
 echo "Building jimmctl..."
 # Build jimmctl so we may add a controller.
@@ -63,16 +39,6 @@ else
     exit 1
 fi
 echo
-echo "Retrieving controller address"
-CONTROLLER_ADDRESS=$(cat "$CONTROLLER_YAML_PATH" | yq '.public-address' |  cut -d ":" -f 1)
-echo "Controller address is: $CONTROLLER_ADDRESS" 
-echo
-echo "Updating $CONTROLLER_YAML_PATH public-address..."
-yq -i e '.public-address |= "juju-apiserver:17070"' "$CONTROLLER_YAML_PATH"
-echo
-echo "Updating containers /etc/hosts..."
-docker compose exec -w /etc --no-TTY jimm bash -c "echo '$CONTROLLER_ADDRESS juju-apiserver' >> hosts"
-echo
 echo "Adding controller from path: $CONTROLLER_YAML_PATH"
 ./jimmctl add-controller "$CONTROLLER_YAML_PATH"
 echo