xos/synchronizers/onboarding/xosbuilder.py

import os
import base64
import jinja2
import string
import sys
import urllib2
import urlparse
import xmlrpclib

from xos.config import Config
from core.models import Service, ServiceController, ServiceControllerResource, LoadableModule, LoadableModuleResource, XOS
from xos.logger import Logger, logging

from django.utils import timezone

logger = Logger(level=logging.INFO)

def add_unique(list, item):
   if not item in list:
       list.append(item)

class XOSBuilder(object):
    UI_KINDS=["models", "admin", "admin_template", "django_library", "rest_service", "rest_tenant", "tosca_custom_types", "tosca_resource","public_key","vendor_js"]
    SYNC_CONTROLLER_KINDS=["synchronizer", "private_key", "public_key"]
    SYNC_ALLCONTROLLER_KINDS=["models", "django_library"]

    def __init__(self):
        self.source_sync_image = "xosproject/xos" # "xosproject/xos-synchronizer-openstack"
        self.build_dir = "/opt/xos/BUILD/"
        self.build_tainted = False

    # stuff that has to do with downloading

    def get_base_dest_dir(self, scr):
        xos_base = "opt/xos"
        service_name = scr.loadable_module.name
        base_dirs = {"models": "%s/services/%s/" % (xos_base, service_name),
                     "admin": "%s/services/%s/" % (xos_base, service_name),
                     "admin_template": "%s/services/%s/templates/" % (xos_base, service_name),
                     "django_library": "%s/services/%s/" % (xos_base, service_name),
                     "synchronizer": "%s/synchronizers/%s/" % (xos_base, service_name),
                     "tosca_custom_types": "%s/tosca/custom_types/" % (xos_base),
                     "tosca_resource": "%s/tosca/resources/" % (xos_base),
                     "rest_service": "%s/api/service/" % (xos_base),
                     "rest_tenant": "%s/api/tenant/" % (xos_base),
                     "private_key": "%s/services/%s/keys/" % (xos_base, service_name),
                     "public_key": "%s/services/%s/keys/" % (xos_base, service_name),
                     "vendor_js": "%s/core/xoslib/static/vendor/" % (xos_base)}
        dest_dir = base_dirs[scr.kind]

        return dest_dir

    def get_dest_dir(self, scr):
        dest_dir = self.get_base_dest_dir(scr)

        if scr.subdirectory:
            dest_dir = os.path.join(dest_dir, scr.subdirectory)

        return dest_dir

    def get_build_fn(self, scr):
        dest_dir = self.get_dest_dir(scr)
        dest_fn = os.path.split(urlparse.urlsplit(scr.full_url).path)[-1]
        return os.path.join(dest_dir, dest_fn)

    def get_download_fn(self, scr):
        dest_fn = self.get_build_fn(scr)
        return os.path.join(self.build_dir, dest_fn)

    def read_manifest(self, scr, fn):
        manifest = []
        manifest_lines = file(fn).readlines()
        manifest_lines = [x.strip() for x in manifest_lines]
        manifest_lines = [x for x in manifest_lines if x]
        for line in manifest_lines:
            url_parts = urlparse.urlsplit(scr.full_url)
            new_path = os.path.join(os.path.join(*os.path.split(url_parts.path)[:-1]),line)
            url = urlparse.urlunsplit( (url_parts.scheme, url_parts.netloc, new_path, url_parts.query, url_parts.fragment) )

            build_fn = os.path.join(self.get_dest_dir(scr), line)
            download_fn = os.path.join(self.build_dir, build_fn)

            manifest.append( (url, download_fn, build_fn) )
        return manifest

    def download_file(self, url, dest_fn):
        logger.info("Download %s to %s" % (url, dest_fn))
        if not os.path.exists(os.path.dirname(dest_fn)):
            os.makedirs(os.path.dirname(dest_fn))
        obj = urllib2.urlopen(url)
        file(dest_fn,"w").write(obj.read())

        # make python files executable
        if dest_fn.endswith(".py"): # and contents.startswith("#!"):
            os.chmod(dest_fn, 0755)

    def download_resource(self, scr):
        if scr.format == "manifest":
            manifest_fn = self.get_download_fn(scr)
            self.download_file(scr.full_url, manifest_fn)
            manifest = self.read_manifest(scr, manifest_fn)
            for (url, download_fn, build_fn) in manifest:
                self.download_file(url, download_fn)
        else:
            self.download_file(scr.full_url, self.get_download_fn(scr))

# XXX docker creates a new container and commits it for every single COPY
# line in the dockerfile. This causes services with many files (for example,
# vsg) to take ~ 10-15 minutes to build the docker file. So instead we'll copy
# the whole build directory, and then run a script that copies the files
# we want.

#    def get_docker_lines(self, scr):
#        if scr.format == "manifest":
#            manifest_fn = self.get_download_fn(scr)
#            manifest = self.read_manifest(scr, manifest_fn)
#            lines = []
#            for (url, download_fn, build_fn) in manifest:
#               script.append("mkdir -p
#               #lines.append("COPY %s /%s" % (build_fn, build_fn))
#            return lines
#        else:
#            build_fn = self.get_build_fn(scr)
#            #return ["COPY %s /%s" % (build_fn, build_fn)]

#    def get_controller_docker_lines(self, controller, kinds):
#        need_service_init_py = False
#        dockerfile=[]
#        for scr in controller.loadable_module_resources.all():
#            if scr.kind in kinds:
#                lines = self.get_docker_lines(scr)
#                dockerfile = dockerfile + lines
#            if scr.kind in ["admin", "models"]:
#                need_service_init_py = True
#
#        if need_service_init_py:
#            file(os.path.join(self.build_dir, "opt/xos/empty__init__.py"),"w").write("")
#            dockerfile.append("COPY opt/xos/empty__init__.py /opt/xos/services/%s/__init__.py" % controller.name)
#
#        return dockerfile

    def get_script_lines(self, scr):
        if scr.format == "manifest":
            manifest_fn = self.get_download_fn(scr)
            manifest = self.read_manifest(scr, manifest_fn)
            lines = []
            for (url, download_fn, build_fn) in manifest:
               lines.append("mkdir -p /%s" % os.path.dirname(build_fn))
               lines.append("cp /build/%s /%s" % (build_fn, build_fn))
            return lines
        else:
            build_fn = self.get_build_fn(scr)
            return ["mkdir -p /%s" % os.path.dirname(build_fn),
                    "cp /build/%s /%s" % (build_fn, build_fn)]

    def get_controller_script_lines(self, controller, kinds):
        need_service_init_py = False
        script=[]
        inits=[]
        for scr in list(controller.loadable_module_resources.all()):
            if not (scr.kind in kinds):
                continue

            # Check and see if the resource we're trying to install has
            # disappeared. This may happen if the onboarding synchronizer
            # container has been destroyed and restarted. In this case, flag
            # the resource for re-download, and set the build_tainted bit
            # so we can throw an exception after we've evaluated all
            # resources.

            download_fn = self.get_download_fn(scr)
            if not os.path.exists(download_fn):
                logger.info("File %s is missing; dirtying the resource" % download_fn)
                scr.backend_status = "2 - download_fn is missing"
                scr.updated = timezone.now()
                scr.save(update_fields=['backend_status', 'updated'])
                self.build_tainted = True
                continue

            lines = self.get_script_lines(scr)
            script = script + lines

            # compute the set of __init__.py files that we will need
            if scr.kind in ["admin", "models", "rest_service", "rest_tenant"]:
                dir = self.get_base_dest_dir(scr)
                add_unique(inits, dir)

                if scr.subdirectory:
                    for part in scr.subdirectory.split("/"):
                        dir = os.path.join(dir, part)
                        add_unique(inits, dir)

        for init in inits:
            script.append("echo > %s" % os.path.join("/",init,"__init__.py"))

        return script

    def check_controller_unready(self, controller):
        unready_resources=[]
        for scr in controller.loadable_module_resources.all():
            if (not scr.backend_status) or (not scr.backend_status.startswith("1")):
                unready_resources.append(scr)

        return unready_resources

    # stuff that has to do with building

    def create_xos_app_data(self, name, script, app_list, migration_list):
        if not os.path.exists(os.path.join(self.build_dir,"opt/xos/xos")):
            os.makedirs(os.path.join(self.build_dir,"opt/xos/xos"))

        if app_list:
            script.append("mkdir -p /opt/xos/xos")
            script.append("cp /build/opt/xos/xos/%s_xosbuilder_app_list /opt/xos/xos/xosbuilder_app_list" % name)
            #dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_app_list /opt/xos/xos/xosbuilder_app_list" % name)
            file(os.path.join(self.build_dir, "opt/xos/xos/%s_xosbuilder_app_list") % name, "w").write("\n".join(app_list)+"\n")

        if migration_list:
            script.append("mkdir -p /opt/xos/xos")
            script.append("cp /build/opt/xos/xos/%s_xosbuilder_migration_list /opt/xos/xos/xosbuilder_migration_list" % name)
            #dockerfile.append("COPY opt/xos/xos/%s_xosbuilder_migration_list /opt/xos/xos/xosbuilder_migration_list" % name)
            file(os.path.join(self.build_dir, "opt/xos/xos/%s_xosbuilder_migration_list") % name, "w").write("\n".join(migration_list)+"\n")

    def create_ui_dockerfile(self):
        self.build_tainted = False
        xos = XOS.objects.all()[0]
        dockerfile_fn = "Dockerfile.UI"

        app_list = []
        migration_list = []

        dockerfile = ["FROM %s" % xos.source_ui_image]
        script = []
        for controller in LoadableModule.objects.all():
            if self.check_controller_unready(controller):
                 logger.warning("Loadable Module %s has unready resources" % str(controller))
                 continue

            #dockerfile = dockerfile + self.get_controller_docker_lines(controller, self.UI_KINDS)
            script = script + self.get_controller_script_lines(controller, self.UI_KINDS)
            if controller.loadable_module_resources.filter(kind="models").exists():
                app_list.append("services." + controller.name)
                migration_list.append(controller.name)

        self.create_xos_app_data("ui", script, app_list, migration_list)

        file(os.path.join(self.build_dir, "install-xos.sh"), "w").write("\n".join(script)+"\n")
        dockerfile.append("COPY . /build/")
        dockerfile.append("RUN bash /build/install-xos.sh")

        file(os.path.join(self.build_dir, dockerfile_fn), "w").write("\n".join(dockerfile)+"\n")

        if self.build_tainted:
            raise Exception("Build was tainted due to errors")

        return {"dockerfile_fn": dockerfile_fn,
                "docker_image_name": "xosproject/xos-ui"}

    def create_synchronizer_dockerfile(self, controller):
        self.build_tainted = False

        if not controller.loadable_module_resources.filter(kind="synchronizer").exists():
            # it doesn't have a synchronizer, therefore it doesn't need a dockerfile
            return None

        # bake in the synchronizer from this controller
        sync_lines = self.get_controller_script_lines(controller, self.SYNC_CONTROLLER_KINDS)

        if self.build_tainted:
            raise Exception("Build was tainted due to errors")

        # If there's no sync_lines for this ServiceController, then it must not
        # have a synchronizer.
        if not sync_lines:
            return None

        dockerfile_fn = "Dockerfile.%s" % controller.name
        dockerfile = ["FROM %s" % self.source_sync_image]
        script = []

        # Now bake in models from this controller as well as the others
        # It's important to bake all services in, because some services'
        # synchronizers may depend on models from another service.
        app_list = []
        for c in LoadableModule.objects.all():
            script = script + self.get_controller_script_lines(c, self.SYNC_ALLCONTROLLER_KINDS)
            if c.loadable_module_resources.filter(kind="models").exists():
                app_list.append("services." + c.name)

        self.create_xos_app_data(controller.name, script, app_list, None)

        script = script + sync_lines

        file(os.path.join(self.build_dir, "install-%s.sh" % controller.name), "w").write("\n".join(script)+"\n")
        dockerfile.append("COPY . /build/")
        dockerfile.append("RUN bash /build/install-%s.sh" % controller.name)

        file(os.path.join(self.build_dir, dockerfile_fn), "w").write("\n".join(dockerfile)+"\n")

        if self.build_tainted:
            raise Exception("Build was tainted due to errors")

        return {"dockerfile_fn": dockerfile_fn,
                "docker_image_name": "xosproject/xos-synchronizer-%s" % controller.name}

    def create_docker_compose(self):
         xos = XOS.objects.all()[0]

         volume_list = []
         for volume in xos.volumes.all():
             volume_list.append({"host_path": volume.host_path,
                                 "container_path": volume.container_path,
                                 "read_only": volume.read_only})

         if xos.extra_hosts:
             extra_hosts = [x.strip() for x in xos.extra_hosts.split(",")]
         else:
             extra_hosts = []

         containers = {}

#         containers["xos_db"] = \
#                            {"image": "xosproject/xos-postgres",
#                             "expose": [5432]}

         containers["xos_ui"] = \
                            {"image": "xosproject/xos-ui",
                             "command": "python /opt/xos/manage.py runserver 0.0.0.0:%d --insecure --makemigrations" % xos.ui_port,
                             "ports": {"%d"%xos.ui_port : "%d"%xos.ui_port},
                             #"links": ["xos_db"],
                             "external_links": ["%s:%s" % (xos.db_container_name, "xos_db")],
                             "extra_hosts": extra_hosts,
                             "volumes": volume_list}

         if xos.no_start:
             containers["xos_ui"]["command"] = "sleep 864000"

#         containers["xos_bootstrap_ui"] = {"image": "xosproject/xos",
#                             "command": "python /opt/xos/manage.py runserver 0.0.0.0:%d --insecure --makemigrations" % xos.bootstrap_ui_port,
#                             "ports": {"%d"%xos.bootstrap_ui_port : "%d"%xos.bootstrap_ui_port},
#                             #"external_links": ["%s:%s" % (xos.db_container_name, "xos_db")],
#                             "links": ["xos_db"],
#                             "volumes": volume_list}

         if not xos.frontend_only:
             for c in ServiceController.objects.all():
                 if self.check_controller_unready(c):
                     logger.warning("Controller %s has unready resources" % str(c))
                     continue

                 if c.loadable_module_resources.filter(kind="synchronizer").exists():
                     if c.synchronizer_run and c.synchronizer_config:
                         command = 'bash -c "sleep 120; cd /opt/xos/synchronizers/%s; python ./%s -C %s"' % (c.name, c.synchronizer_run, c.synchronizer_config)
                     else:
                         command = 'bash -c "sleep 120; cd /opt/xos/synchronizers/%s; bash ./run.sh"' % c.name

                     containers["xos_synchronizer_%s" % c.name] = \
                                    {"image": "xosproject/xos-synchronizer-%s" % c.name,
                                     "command": command,
                                     "external_links": ["%s:%s" % (xos.db_container_name, "xos_db")],
                                     "extra_hosts": extra_hosts,
                                     "volumes": volume_list}

                     if c.no_start:
                         containers["xos_synchronizer_%s" % c.name]["command"] = "sleep 864000"

         vars = { "containers": containers }

         template_loader = jinja2.FileSystemLoader( "/opt/xos/synchronizers/onboarding/templates/" )
         template_env = jinja2.Environment(loader=template_loader)
         template = template_env.get_template("docker-compose.yml.j2")
         buffer = template.render(vars)

         if not os.path.exists("/opt/xos/synchronizers/onboarding/docker-compose"):
             os.makedirs("/opt/xos/synchronizers/onboarding/docker-compose")
         file("/opt/xos/synchronizers/onboarding/docker-compose/docker-compose.yml", "w").write(buffer)

#    def build_xos(self):
#        dockerfiles=[]
#        dockerfiles.append(self.create_ui_dockerfile())
#
#        for controller in ServiceController.objects.all():
#            dockerfiles.append(self.create_synchronizer_dockerfile(controller))