Prepare containers for Docker Hub release (#86)

* Prepare containers for Docker Hub release - Fix builds from Focal thru Noble - Forward args across Earthfiles - Allow tag renaming Signed-off-by: Michel Hidalgo <[email protected]> * Fix devel targets Signed-off-by: Michel Hidalgo <[email protected]> * Avoid --pass-args in Earthfiles Workaround earthly/earthly#4162 by explicitly forwarding arguments (not great, no options) Signed-off-by: Michel Hidalgo <[email protected]> * Adjust lambkin Python packages for Ubuntu Noble Signed-off-by: Michel Hidalgo <[email protected]> * One more fix for Noble Signed-off-by: Michel Hidalgo <[email protected]> * Fix broken benchmarks Signed-off-by: Michel Hidalgo <[email protected]> * Complete steps in documentation * Adjust entrpoints and labels Signed-off-by: Michel Hidalgo <[email protected]> * Fix root devcontainer Signed-off-by: Michel Hidalgo <[email protected]> * Mention the need for LFS to get the beluga datasets * Use nav2 from the distribution * Fix documentation for beluga_vs_nav2 testbench * Update release image labels Signed-off-by: Michel Hidalgo <[email protected]> --------- Signed-off-by: Michel Hidalgo <[email protected]> Co-authored-by: Gerardo Puga <[email protected]>
Ekumen-OS · Jul 17, 2024 · f0ef346 · f0ef346
1 parent 4c95441
commit f0ef346
Show file tree

Hide file tree

Showing 21 changed files with 859 additions and 461 deletions.
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -1,7 +1,7 @@
 services:
   devcontainer:
     image: ekumenlabs/lambkin:dev
-    container_name: lambkin-dev
+    container_name: lambkin-devel
     environment:
       - DISPLAY
       - QT_X11_NO_MITSHM=1

diff --git a/.earthlyignore b/.earthlyignore
@@ -0,0 +1 @@
+tools
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,9 @@
-# catkin
+# catkin / colcon
 build
 install
 devel
 logs
+log
 
 # python
 **/__pycache__

diff --git a/Earthfile b/Earthfile
@@ -18,38 +18,57 @@ IMPORT ./src/external/os AS os
 ARG --global components="core/lambkin-shepherd core/lambkin-clerk external/ros2 external/typesetting/latex external/profiling/timemory"
 
 embed-ubuntu-devel:
-    ARG distro=jammy
+    ARG distro
+    ARG rosdistro  # forward
     FROM os+ubuntu --distro=${distro}
     FOR component IN ${components}
-        MERGE os+ubuntu.../src/${component}+embed-ubuntu-devel --distro=${distro}
+        MERGE os+ubuntu.../src/${component}+embed-ubuntu-devel --distro=${distro} --rosdistro=${rosdistro}
     END
     RUN echo "echo 'Logged into development environment for ${components} on Ubuntu: ${distro}'" > /etc/profile.d/banner.sh
 
 embed-ubuntu-release:
-    ARG distro=jammy
+    ARG distro
+    ARG rosdistro  # forward
     FROM os+ubuntu --distro=${distro}
     FOR component IN ${components}
-        MERGE os+ubuntu.../src/${component}+embed-ubuntu-release --distro=${distro}
+        MERGE os+ubuntu.../src/${component}+embed-ubuntu-release --distro=${distro} --rosdistro=${rosdistro}
     END
 
 ubuntu-devel:
-    ARG distro=jammy
     ARG user=lambkin
     ARG uid=1000
     ARG gid=1000
-    FROM +embed-ubuntu-devel --distro=${distro}
-    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid}
-    SAVE IMAGE ekumenlabs/lambkin:ubuntu-${distro}-dev ekumenlabs/lambkin:dev
+    ARG distro=jammy
+    ARG rodistro  # forward
+    ARG tag=ubuntu-${distro}-dev
+    FROM +embed-ubuntu-devel --distro=${distro}  --rosdistro=${rosdistro}
+    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid} --workdir=/workspace
+    SAVE IMAGE ekumenlabs/lambkin:${tag} ekumenlabs/lambkin:dev
 
 local-ubuntu-devel:
     LOCALLY
     ARG distro=jammy
-    LET user = "$(whoami)"
-    LET uid = "$(id -u)"
-    LET gid = "$(id -g)"
-    BUILD +ubuntu-devel --distro=${distro} --user=${user} --uid=${uid} --gid=${gid}
+    ARG rosdistro  # forward
+    LET user="$(whoami)"
+    LET uid="$(id -u)"
+    LET gid="$(id -g)"
+    BUILD +ubuntu-devel --distro=${distro} --rosdistro=${rosdistro} \
+                        --user=${user} --uid=${uid} --gid=${gid}
 
 ubuntu-release:
     ARG distro=jammy
-    FROM +embed-ubuntu-release --distro=${distro}
-    SAVE IMAGE ekumenlabs/lambkin:ubuntu-${distro}
+    ARG rosdistro  # forward
+    ARG tag=ubuntu-${distro}
+    FROM +embed-ubuntu-release --distro=${distro} --rosdistro=${rosdistro}
+    ARG EARTHLY_GIT_BRANCH
+    ARG EARTHLY_GIT_HASH
+    LABEL "org.opencontainers.image.ref.name"="${EARTHLY_GIT_BRANCH}"
+    LABEL "org.opencontainers.image.version"="${tag}"
+    LABEL "org.opencontainers.image.title"="LAMBKIN"
+    LABEL "org.opencontainers.image.description"="Localization And Mapping BenchmarKINg toolkit release, bundling ${components} components"
+    LABEL "org.opencontainers.image.revision"="${EARTHLY_GIT_HASH}"
+    LABEL "org.opencontainers.image.source"="https://github.com/ekumenlabs/lambkin"
+    LABEL "org.opencontainers.image.base.name"="ubuntu:${distro}"
+    LABEL "org.opencontainers.image.vendor"="Ekumen, Inc."
+    LABEL "org.opencontainers.image.licenses"="Apache-2.0"
+    SAVE IMAGE ekumenlabs/lambkin:${tag}
diff --git a/src/benchmarks/beluga_vs_nav2/Earthfile b/src/benchmarks/beluga_vs_nav2/Earthfile
@@ -14,49 +14,48 @@
 
 VERSION 0.8
 
-IMPORT ../../external/ros AS ros
+IMPORT ../../external/os AS os
 IMPORT ../../.. AS lambkin
 
 devel:
+    ARG distro=jammy
+    ARG rosdistro  # forward
     ARG user=lambkin
     ARG uid=1000
     ARG gid=1000
-    ARG distro=jammy
-    FROM lambkin+embed-ubuntu-devel --distro=${distro} \
-         --components="core/lambkin-shepherd core/lambkin-clerk external/ros2
-                       external/typesetting/latex external/profiling/timemory"
+    FROM lambkin+embed-ubuntu-devel --distro=${distro}  --rosdistro=${rosdistro}
     RUN mkdir -p /workspace/src
     WORKDIR /workspace
     RUN cd src && git clone https://github.com/Ekumen-OS/beluga
-    RUN cd src && git clone --depth 1 --no-checkout https://github.com/ros-planning/navigation2.git && \
-        cd navigation2 && git sparse-checkout set nav2_amcl && git checkout
     COPY package.xml src/lambkin/benchmarks/beluga_vs_nav2/package.xml
     RUN . /etc/profile && apt update && rosdep update && \
-        rosdep install -y -i --from-paths src && \
+        rosdep install -y -i --from-paths src \
+            --skip-keys 'lambkin-shepherd lambkin-clerk' && \
         apt clean && rm -rf /var/lib/apt/lists/*
     RUN pip install linuxdoc sphinxcontrib.datatemplates sphinxcontrib-repl
-    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid}
+    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid} --workdir=/workspace
     SAVE IMAGE ekumenlabs/beluga-vs-nav2:dev
 
 local-devel:
     LOCALLY
     ARG distro=jammy
-    LET user = "$(whoami)"
-    LET uid = "$(id -u)"
-    LET gid = "$(id -g)"
-    BUILD +devel --distro=${distro} --user=${user} --uid=${uid} --gid=${gid}
+    ARG rosdistro  # forward
+    LET user="$(whoami)"
+    LET uid="$(id -u)"
+    LET gid="$(id -g)"
+    BUILD +devel --distro=${distro} --rosdistro=${rosdistro} --user=${user} --uid=${uid} --gid=${gid}
 
 build:
     ARG distro=jammy
-    FROM lambkin+ubuntu-devel --distro=${distro} --components="external/ros2"
+    ARG rosdistro  # forward
+    FROM lambkin+ubuntu-devel --distro=${distro} --rosdistro=${rosdistro} --components="external/ros2"
     RUN mkdir -p /workspace/src
     WORKDIR /workspace
     RUN cd src && git clone https://github.com/Ekumen-OS/beluga
-    RUN cd src && git clone --depth 1 --no-checkout https://github.com/ros-planning/navigation2.git && \
-        cd navigation2 && git sparse-checkout set nav2_amcl && git checkout
     COPY . src/beluga_vs_nav2
     RUN . /etc/profile && apt update && rosdep update && \
-        rosdep install -y -i --from-paths src -t build -t buildtool -t test && \
+        rosdep install -y -i --from-paths src -t build -t buildtool -t test \
+            --skip-keys 'lambkin-shepherd lambkin-clerk' && \
         apt clean && rm -rf /var/lib/apt/lists/*
     RUN . /etc/profile && colcon build --merge-install --install-base /opt/ros/application
     LET content = "
@@ -72,12 +71,12 @@ build:
 
 release:
     ARG distro=jammy
-    FROM lambkin+embed-ubuntu-release --distro=${distro} \
-         --components="core/lambkin-shepherd core/lambkin-clerk external/ros2
-                       external/typesetting/latex external/profiling/timemory"
-    COPY (+build/application --distro=${distro}) /opt/ros/application
+    ARG rosdistro  # forward
+    FROM lambkin+embed-ubuntu-release --distro=${distro} --rosdistro=${rosdistro} \
+    COPY (+build/application --distro=${distro} --rosdistro=${rosdistro}) /opt/ros/application
     RUN . /etc/profile && apt update && rosdep update && \
-        rosdep install -i -y --from-path /opt/ros/application -t exec && \
+        rosdep install -i -y --from-path /opt/ros/application \
+            -t exec --skip-keys 'lambkin-shepherd lambkin-clerk' && \
         apt clean && rm -rf /var/lib/apt/lists/*
     RUN pip install linuxdoc sphinxcontrib.datatemplates sphinxcontrib-repl
     ENTRYPOINT ["/bin/bash", "--login", "/opt/ros/application/entrypoint.bash"]

diff --git a/src/benchmarks/beluga_vs_nav2/README.md b/src/benchmarks/beluga_vs_nav2/README.md
@@ -11,8 +11,9 @@ cd {LAMBKIN_ROOT_DIR}
 ./tools/setup.sh
 ./tools/earthly ./src/benchmarks/beluga_vs_nav2+local-devel
 ```
-- Clone `beluga-datasets`
+- Clone `beluga-datasets`. **This repository uses LFS**, so make sure you have it installed.
 ```sh
+sudo apt install git-lfs
 cd src/benchmarks/beluga_vs_nav2
 mkdir -p playground && cd playground
 git clone [email protected]:Ekumen-OS/beluga-datasets.git
@@ -26,18 +27,20 @@ source install/setup.bash
 - Cd to `playgrounds` dir, where you've downloaded `beluga-datasets`
 
 ```sh
-cd src/beluga_vs_nav2/playground
+cd src/lambkin/benchmarks/beluga_vs_nav2/playground
 ```
 
 - Run the benchmark itself.
 ```sh
-ros2 run beluga_vs_nav2 beluga_vs_nav2.robot
+ros2 run beluga_vs_nav2 nominal.robot
+ros2 run beluga_vs_nav2 ndt_swept.robot
+ros2 run beluga_vs_nav2 swept.robot
 ```
-- Inspect the results
+- Inspect the results using the same name of the launch file. For example, if you ran `nominal.robot`, you can inspect the results by running:
 ```sh
-ls beluga_vs_nav2
+ls nominal
 ```
-- Find the report
+- The report can be found in the `report` directory. For example, if you ran `nominal.robot`, you can inspect the `pdf` report by running:
 ```sh
-ls beluga_vs_nav2/report/latex/report.pdf 
+ls nominal/report/build/latex/report.pdf
 ```
diff --git a/src/benchmarks/beluga_vs_nav2/package.xml b/src/benchmarks/beluga_vs_nav2/package.xml
@@ -9,15 +9,21 @@
   <license>Apache-2.0</license>
 
   <buildtool_depend>ament_cmake</buildtool_depend>
-  <depend>launch_ros</depend>
+  <exec_depend>launch_ros</exec_depend>
+
+  <exec_depend>lambkin-clerk</exec_depend>
+  <exec_depend>lambkin-shepherd</exec_depend>
 
   <exec_depend>python3-matplotlib</exec_depend>
   <exec_depend>python3-numpy</exec_depend>
   <exec_depend>python3-pandas</exec_depend>
+  <exec_depend>python3-tk</exec_depend>
+
   <exec_depend>nav2_amcl</exec_depend>
   <exec_depend>nav2_lifecycle_manager</exec_depend>
+
   <exec_depend>beluga_amcl</exec_depend>
-  <exec_depend>python3-tk</exec_depend>
+
   <exec_depend>rosbag2_storage_default_plugins</exec_depend>
 
   <export>

diff --git a/src/benchmarks/cartographer_ros_benchmarks/Earthfile b/src/benchmarks/cartographer_ros_benchmarks/Earthfile
@@ -14,7 +14,7 @@
 
 VERSION 0.8
 
-IMPORT ../../external/ros AS ros
+IMPORT ../../external/os AS os
 IMPORT ../../.. AS lambkin
 
 ARG --global cartographer_rosinstall_uri=https://raw.githubusercontent.com/cartographer-project/cartographer_ros/master/cartographer_ros.rosinstall
@@ -24,25 +24,24 @@ devel:
     ARG uid=1000
     ARG gid=1000
     FROM lambkin+embed-ubuntu-devel --distro=focal \
-         --components="core/lambkin-shepherd core/lambkin-clerk external/ros
-                       external/typesetting/latex external/profiling/timemory"
+         --components="core/lambkin-shepherd core/lambkin-clerk external/ros external/typesetting/latex external/profiling/timemory"
     RUN mkdir -p /workspace/src
     WORKDIR /workspace
     RUN curl -sSL ${cartographer_rosinstall_uri} -o cartographer_ros.rosinstall
     RUN wstool init src cartographer_ros.rosinstall
     COPY package.xml src/lambkin/benchmarks/cartographer_ros_benchmarks/package.xml
     RUN . /etc/profile && apt update && apt install -y libgmock-dev ninja-build stow && \
         ./src/cartographer/scripts/install_abseil.sh && rosdep update && \
-        rosdep install -y -i --from-paths src --skip-keys libabsl-dev && \
+        rosdep install -y -i --from-paths src --skip-keys "libabsl-dev lambkin-shepherd" && \
         apt clean && rm -rf /var/lib/apt/lists/*
-    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid}
+    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid} --workdir=/workspace
     SAVE IMAGE ekumenlabs/cartographer-ros-benchmarks:dev
 
 local-devel:
     LOCALLY
-    LET user = "$(whoami)"
-    LET uid = "$(id -u)"
-    LET gid = "$(id -g)"
+    LET user="$(whoami)"
+    LET uid="$(id -u)"
+    LET gid="$(id -g)"
     BUILD +devel --user=${user} --uid=${uid} --gid=${gid}
 
 build:
@@ -54,7 +53,8 @@ build:
     COPY . src/cartograph_ros_benchmarks
     RUN . /etc/profile && apt update && apt install -y libgmock-dev ninja-build stow && \
         ./src/cartographer/scripts/install_abseil.sh && rosdep update && \
-        rosdep install -y -i --from-paths src -t build -t buildtool -t test --skip-keys "libabsl-dev" && \
+        rosdep install -y -i --from-paths src -t build -t buildtool -t test \
+            --skip-keys "libabsl-dev lambkin-shepherd" && \
         apt clean && rm -rf /var/lib/apt/lists/*
     RUN . /opt/ros/noetic/setup.sh && \
         catkin_make_isolated --install-space /opt/ros/application --install --merge
@@ -79,4 +79,5 @@ release:
             -t exec --skip-keys "libabsl-dev lambkin-shepherd" && \
         apt clean && rm -rf /var/lib/apt/lists/*
     ENTRYPOINT ["/bin/bash", "--login", "/opt/ros/application/entrypoint.bash"]
+    CMD []
     SAVE IMAGE ekumenlabs/cartographer-ros-benchmarks:latest
diff --git a/src/benchmarks/slam_toolbox_benchmarks/Earthfile b/src/benchmarks/slam_toolbox_benchmarks/Earthfile
@@ -30,14 +30,14 @@ devel:
     RUN . /etc/profile && apt update && rosdep update && \
         rosdep install -i -y --from-path src --skip-keys "lambkin-shepherd" && \
         apt clean && rm -rf /var/lib/apt/lists/*
-    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid}
+    DO os+ADDUSER --user=${user} --uid=${uid} --gid=${gid} --workdir=/workspace
     SAVE IMAGE ekumenlabs/slam-toolbox-benchmarks:dev
 
 local-devel:
     LOCALLY
-    LET user = "$(whoami)"
-    LET uid = "$(id -u)"
-    LET gid = "$(id -g)"
+    LET user="$(whoami)"
+    LET uid="$(id -u)"
+    LET gid="$(id -g)"
     BUILD +devel --user=${user} --uid=${uid} --gid=${gid}
 
 build:
@@ -72,4 +72,5 @@ release:
             -t exec --skip-keys "lambkin-shepherd" && \
         apt clean && rm -rf /var/lib/apt/lists/*
     ENTRYPOINT ["/bin/bash", "--login", "/opt/ros/application/entrypoint.bash"]
+    CMD []
     SAVE IMAGE ekumenlabs/slam-toolbox-benchmarks:latest
diff --git a/src/core/lambkin-clerk/Earthfile b/src/core/lambkin-clerk/Earthfile
@@ -25,15 +25,15 @@ build:
 
 embed-ubuntu-devel:
     ARG distro
-    FROM os+ubuntu --distro=$distro
+    FROM os+ubuntu --distro=${distro}
     COPY requirements.txt pyproject.toml poetry.lock .
     RUN pip install -r requirements.txt
     RUN poetry config virtualenvs.create false
     RUN poetry install --no-root --with dev
 
 embed-ubuntu-release:
     ARG distro
-    FROM os+ubuntu --distro=$distro
+    FROM os+ubuntu --distro=${distro}
     RUN apt update && apt install -y cpuinfo util-linux && \
         apt clean && rm -rf /var/lib/apt/lists/*
     COPY +build/sdist/*.whl /tmp/.