Detect module dependencies

dbuild/build.py

@@ -30,6 +30,7 @@
 
 from dbuild.docker_utils import list_modules
 from dbuild.verb import verbs, verb_arguments, VerbException
+from dbuild.dependency import get_dependencies
 
 WORKER_STATUS_POLL_WAIT = 0.5
 
@@ -381,7 +382,7 @@ def main():
         logging.root.setLevel(logging.DEBUG)
 
     arguments.verbs = filter(lambda v: v in verbs.keys(), arguments.args)
-    arguments.modules = filter(lambda m: m in modules, arguments.args)
+    arguments.modules = get_modules(modules, arguments.args)
     logger.info('Modules: %r', arguments.modules)
 
     reserved = arguments.verbs + arguments.modules
@@ -420,5 +421,31 @@ def main():
     execute_plans(plans, arguments.workers)
 
 
+def get_modules(modules, args):
+    mods = filter(lambda m: m in modules, args)
+    raw_lineup = []
+
+    # work on the copy as we will modify the mods
+    # effectively below
+    for i, m in enumerate(mods[:]):
+        logger.info('Analyzing dependencies of %s module' % m)
+        dependencies = get_dependencies(m, mods)
+
+        # if dependencies contain one of the modules that are being
+        # built right now, add them to the mods_map, we will use the
+        # knowledge to lineup the final list
+        active_deps = set(dependencies).intersection(set(modules))
+        if active_deps:
+            raw_lineup.extend(active_deps.extend([m]))
+
+    actual_modules = []
+    for m in raw_lineup:
+        if m not in actual_modules:
+            # append to actual_modules only if module has not been seen
+            actual_modules.append(m)
+
+    return actual_modules
+
+
 if __name__ == '__main__':
     main()

dbuild/dependency.py

@@ -0,0 +1,39 @@
+import logging
+import os
+
+import dockerfile_parse as d_parser
+
+logger = logging.getLogger(__name__)
+
+_BASE_PATH = os.path.realpath(os.getcwd())
+
+
+def get_dependencies(image, all_images):
+    deps = []
+    dockerfile = get_dockerfile(image)
+
+    if not dockerfile:
+        logging.warning('%s is not image home or '
+                        'something went terribly wrong' % image)
+        raise deps
+
+    parent_image = dockerfile.baseimage
+    # note(kornicameister) check if the parent image is part of the all modules
+    # if so, traverse even deeper to reach
+    if parent_image in all_images:
+        deps.extend(get_dependencies(parent_image, all_images))
+
+    # note(kornicameister) reverse the dependencies, we want to
+    # have the grand elder to always be the first module that we build
+
+    return deps[::-1]
+
+
+def get_dockerfile(image):
+    path_to_file = os.path.join(_BASE_PATH, image, 'Dockerfile')
+    if os.path.exists(path_to_file):
+        return d_parser.DockerfileParser(
+            path=path_to_file,
+            cache_content=True
+        )
+    return None

requirements.txt

@@ -9,3 +9,4 @@ attrs
 futures
 tqdm
 docker
+https://github.com/DBuildService/dockerfile-parse.git@master#egg=dockerfile-parse