bigtop.bom

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/** DSL documentation
 The structure of this configuration DSL is pretty self-explanatory. The main
 section is 'components' that contains the description of all things included
 into the stack. The following words are considered terminals and shouldn't be
 modified as it will affect the parsing behavior:
   - bigtop
   - version
   - stack
   - version_base
   - apache
   - pkg
   - tarball, source, destinations
   - url, download_path, site, archive
 Also, the following are base constants and if renamed will affect the some of
 the stack's definitions
   - APACHE_MIRROR
   - APACHE_ARCHIVE

 bigtop { // *the name should be change: the parsing code depends on it*
   version = "STACK-VERSION" // *required*
   stack { // *required* Fundamental properties of the Stack: JDK, SDK, GDK, etc
     'jdk' { version = '1.8'; version_base = version }
     'scala' { version = '2.10.4'; version_base = version }
   }
   apache { // *required* These shoudn't be modified unless ASF Infra demands changes
     APACHE_MIRROR = "https://apache.osuosl.org"
     APACHE_ARCHIVE = "https://archive.apache.org/dist"
   }
   git { // *optional* This is a global setting to access protected git repositories, 
         //            can be specified per component as well
     user = "john_doe"
     token = "john's access token"
   }
   components { *required; preserve the name* if empty, nothing will be built
     'label' { // label *SHOULD* be the same as the name; otherwise some tasks will fail
       name    = 'component1' // *required* the name of the component
       // 'pkg' value is optional and will be set to that of 'name' i.e. [pkg := name]
       pkg     = name // *optional* and will be set to the 'name' value
       // 'base' is required; [pkg := base ]; [release := 1 ]
       version { base  = 'x.y.z'; pkg   = base; release = 1 }
       tarball {
         source      = "apache-component1-${version.base}.tar.gz"
         // It is advised to use different destination filenames to avoid
         // clashes when working with git repos and downloading the artifacts
         // from the branches with the same names.
         destination = source
       }
       url { // *optional*
         download_path = "/component1/component1-${version.base}"
         site          = "${apache.APACHE_MIRROR}/${download_path}"
         archive       = "${apache.APACHE_ARCHIVE}/${download_path}"
       }
       git {
         // Setting the info to access a git repository. Ref is any valid git reference.
         // If git repo information is provided, the *url* element above will be ignored.
         repo  = "https://github.com/apache/bigtop.git"
         ref   = "branch-name"
         // *dir* defines the name of the top-level folder inside of the tar-ball archive.
         // if set to null, the directory name will be set to tar-ball.dist without
         // the .tar* suffix
         // This setting is important to allow build to locate unpacked source code
         dir   = "${name}-${version.base}-src"
         // *optional*
         // You can setup repo-specific user credentials overriding any global settings
         user  = "john_doe"
         token = "john's access token"
       }
     }
   }
 }
 End of DSL Documentation */

bigtop {
/** Base Configuration of the mirror and archives */
  version = "1.6.0-SNAPSHOT"
  stack {
    'jdk' { version = "1." + ( System.getenv('BIGTOP_JDK') ?: "8" ); version_base = version }
    'scala' { version = '2.12.13'; version_base = version }
  }
  apache {
    APACHE_MIRROR = "https://apache.osuosl.org"
    APACHE_ARCHIVE = "https://archive.apache.org/dist"
  }
/** End of Base Configuration */

  /** Dependencies section defines the order in which packages should be built.
      The syntax is as follows:
           dependsOn := [list of dependents]
      So, below phoenix and giraph ought to be build after the hbase has been built

      To build a component with its dependencies, run build with -Dbuildwithdeps=true
  */
  dependencies = [
    'bigtop-utils':['ambari', 'bigtop-groovy', 'bigtop-jsvc', 'bigtop-tomcat',
                    'elasticsearch', 'flink', 'logstash', 'qfs', 'zookeeper'],
    'bigtop-groovy':['hadoop'],
    'bigtop-jsvc':['hadoop'],
    'bigtop-tomcat':['oozie', 'sqoop2'],
    zookeeper:['hadoop', 'hbase', 'kafka'],
    hadoop:['ignite-hadoop', 'hbase', 'hive', 'tez', 'sqoop', 'sqoop2',
      'oozie', 'mahout', 'flume', 'giraph', 'solr', 'spark',
      'phoenix', 'alluxio', 'ycsb', 'zeppelin'
    ],
    hbase:['phoenix','giraph','ycsb','hive'],
    hive:['oozie', 'zeppelin'],
    'ignite-hadoop':['zeppelin'],
    spark:['livy', 'mahout', 'oozie', 'zeppelin'],
    flink:['mahout']
  ]

  components {
    'zookeeper' {
      name    = 'zookeeper'
      pkg     = name
      version {
        base  = '3.4.14'
        pkg   = base
        release = 1
      }
      tarball {
        source      = "zookeeper-${version.base}.tar.gz"
        destination = source
      }
      url {
        download_path = "/zookeeper/zookeeper-${version.base}"
        site          = "${apache.APACHE_MIRROR}/${download_path}"
        archive       = "${apache.APACHE_ARCHIVE}/${download_path}"
      }
    }
    'hadoop' {
      name    = 'hadoop'
      relNotes = 'Apache Hadoop'
      version { base = '3.2.2'; pkg = base; release = 1 }
      tarball { destination = "${name}-${version.base}.tar.gz"
                source      = "${name}-${version.base}-src.tar.gz" }
      url     { download_path = "/$name/common/$name-${version.base}"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'ignite-hadoop' {
      name    = 'ignite-hadoop'
      relNotes = 'Apache Ignite in-memory data fabric'
      version { base = '2.7.6'; pkg = base; release = 1 }
      tarball { destination = "${name}-${version.base}.tar.gz"
                source      = "apache-ignite-${version.base}-src.zip" }
      url     { download_path = "/ignite/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }

    'hbase' {
      name    = 'hbase'
      relNotes = 'Apache HBase'
      version { base = '2.2.6'; pkg = base; release = 1 }
      tarball { destination = "${name}-${version.base}.tar.gz"
                source      = "${name}-${version.base}-src.tar.gz" }
      url     { download_path = "/$name/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }

    'hive' {
      name    = 'hive'
      relNotes = 'Apache Hive'
      version { base = '3.1.2'; pkg = base; release = 1 }
      tarball { destination = "apache-${name}-${version.base}-src.tar.gz"
                source      = destination }
      url     { download_path = "/$name/$name-${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'tez' {
      name    = 'tez'
      relNotes = 'Apache TEZ'
      version { base = '0.10.0'; pkg = base; release = 1 }
      tarball { destination = "apache-${name}-${version.base}-src.tar.gz"
                source      = destination }
      url     { download_path = "/$name/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'sqoop' {
      name    = 'sqoop'
      relNotes = 'Apache Sqoop v1'
      version { base = '1.4.7'; pkg = base; release = 1 }
      tarball { destination = "${name}-${version.base}.tar.gz"
                source      = destination }
      url     { download_path = "/$name/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'sqoop2' {
      name    = 'sqoop2'
      relNotes = 'Apache Sqoop v2'
      version { base = '1.99.4'; pkg = base; release = 1 }
      tarball { destination = "${sqoop.name}-${version.base}.tar.gz"
                source      = destination }
      url     { download_path = "/${sqoop.name}/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'oozie' {
      name    = 'oozie'
      relNotes = 'Apache Oozie'
      version { base = '4.3.0'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = destination }
      url     { download_path = "/$name/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'mahout' {
      name    = 'mahout'
      relNotes = 'Apache Mahout'
      version { base = '0.13.0'; pkg = base; release = 1 }
      tarball { destination = "apache-$name-distribution-${version.base}-src.tar.gz"
                source      = destination }
      url     { download_path = "/$name/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'flume' {
      name    = 'flume'
      relNotes = 'Apache Flume'
      version { base = '1.9.0'; pkg = base; release = 1 }
      tarball { destination = "apache-$name-${version.base}-src.tar.gz"
                source      = destination }
      url     { download_path = "/$name/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'giraph' {
      name    = 'giraph'
      relNotes = 'Apache Giraph'
      version { base = '1.2.0'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "$name-dist-${version.base}-src.tar.gz" }
      url     { download_path = "/$name/$name-${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'solr' {
      name    = 'solr'
      relNotes = 'Apache Solr'
      version { base = '6.6.6'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}-src.tgz"
                source      = destination }
      url     { download_path = "lucene/$name/${version.base}"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'spark' {
      name    = 'spark'
      pkg     = 'spark-core'
      relNotes = 'Apache Spark'
      /*
       * Remember to fetch the new LICENSE-binary and licenses-binary files
       * when upgrading spark version.
       * See comments in [bigtop-packages/src/common/spark/install_spark.sh] for details.
       */
      version { base = '3.0.1'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "$name-${version.base}.tgz" }
      url     { download_path = "/$name/$name-${version.base}"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }

    'flink' {
      name    = 'flink'
      relNotes = 'Apache Flink'
      version { base = '1.11.3'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "$name-${version.base}-src.tgz" }
      url     { download_path = "/$name/$name-${version.base}"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }


    'phoenix' {
      name    = 'phoenix'
      relNotes = 'Apache Phoenix: A SQL skin over HBase'
      version { base = "5.1.0"; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}-src.tar.gz"
                source      = destination }
      url     { download_path = "/$name/$name-${version.base}"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'bigtop-groovy' {
      name    = 'bigtop-groovy'
      version { base = '2.5.4'; pkg = '2.5.4'; release = 1}
      relNotes = "Groovy: a dynamic language for the Java platform"
      tarball { destination = "$name-${version.base}.tar.gz";
                source      = "apache-groovy-binary-${version.base}.zip"}
      url     { site = "https://dl.bintray.com/groovy/maven/"; archive = site }
      // Optional, as only null values are specified
      git     { repo = null; ref = null; dir = null}
    }
    'bigtop-utils' {
      name    = "bigtop-utils"
      relNotes = "Service package for Apache Bigtop runtime"
      version { base = bigtop.version; pkg = base-"-SNAPSHOT"; release = 1 }
      tarball { destination = "bigtop-utils-${version.base}.tar.gz" }
    }
    'bigtop-jsvc' {
      name    = "bigtop-jsvc"
      relNotes = "Apache Common Daemon (jsvc) service package"
      version { base = '1.0.15'; pkg = base; release = 1 }
      tarball { source      = "commons-daemon-${version.base}-native-src.tar.gz"
                destination = "commons-daemon-${version.base}.tar.gz" }
      url     { download_path = "/commons/daemon/source"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'bigtop-tomcat' {
      name    = "bigtop-tomcat"
      relNotes = "Apache Tomcat"
      version { base = '8.5.57'; pkg = base; release = 1 }
      tarball { source      = "apache-tomcat-${version.base}-src.tar.gz"
                destination = "apache-tomcat-${version.base}.tar.gz" }
      url     { download_path = "/tomcat/tomcat-8/v${version.base}/src"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'alluxio' {
      name    = "alluxio"
      pkg     = "alluxio"
      relNotes = "Alluxio: a memory-centric distributed file system"
      version { base = '2.4.1'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "v${version.base}.tar.gz" }
      url     { site = "https://github.com/Alluxio/alluxio/archive"
                archive = site }
    }
    'kafka' {
      name    = 'kafka'
      relNotes = 'Apache Kafka'
      version { base = '2.4.1'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "$name-${version.base}-src.tgz" }
      url     { download_path = "/$name/${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'ycsb' {
      name    = 'ycsb'
      relNotes = 'Yahoo! Cloud Serving Benchmark'
      version { base = '0.17.0'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "${version.base}.tar.gz" }
      url     { site = "https://github.com/brianfrankcooper/YCSB/archive"
                archive = site }
    }
    'zeppelin' {
      name    = 'zeppelin'
      relNotes = 'Apache Zeppelin'
      version { base = '0.9.0'; pkg = base; release = 1 }
      tarball { source      = "$name-${version.base}.tgz"
                destination = "$name-${version.base}.tar.gz" }
      url     { download_path = "/$name/$name-${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'qfs' {
      name    = 'qfs'
      relNotes = 'Quantcast Filesystem'
      version { base = '2.0.0'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "${version.base}.tar.gz" }
      url     { site = "https://github.com/quantcast/qfs/archive"
                archive = site }
    }
    'gpdb' {
      name    = 'gpdb'
      relNotes = 'GreenPlum'
      version { base = '5.10.0'; pkg = '5.10.0'; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "${version.base}.tar.gz" }
      url     { site = "https://github.com/greenplum-db/gpdb/archive/"
                archive = site }
    }
    'ambari' {
      name    = 'ambari'
      relNotes = 'Apache Ambari'
      version { base = '2.6.1'; pkg = '2.6.1.0'; release = 1 }
      tarball { destination = "apache-$name-${version.base}-src.tar.gz"
                source      = destination }
      url     { download_path = "/$name/$name-${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'bigtop-ambari-mpack' {
      name    = 'bigtop-ambari-mpack'
      relNotes = 'Bigtop Mpack'
      version { base = '2.6.1'; pkg = '2.6.1.0'; release = 1 }
      tarball { destination = "apache-ambari-${version.base}-src.tar.gz"
                source      = destination }
      url     { download_path = "/ambari/ambari-${version.base}/"
                site = "${apache.APACHE_MIRROR}/${download_path}"
                archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
    }
    'livy' {
      name    = 'livy'
      relNotes = 'Apache Livy'
      version { base = '0.7.0'; pkg = base; release = 1 }
      tarball { destination = "${name}-${version.base}.tar.gz"
                source      = destination  }
      git {
        repo = "https://github.com/apache/incubator-livy.git"
        ref = "v${version.base}-incubating"
        dir  = "${name}-${version.pkg}"
      }
    }
    'elasticsearch' {
      name    = 'elasticsearch'
      relNotes = 'Search and Analytics engine'
      version { base = '5.6.14'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "v${version.base}.zip" }
      url     { site = "https://github.com/elastic/elasticsearch/archive"
                archive = site }
    }
    'logstash' {
      name    = 'logstash'
      relNotes = 'Data processing pipeline'
      version { base = '5.4.1'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "v${version.base}.zip" }
      url     { site = "https://github.com/elastic/logstash/archive"
                archive = site }
    }
    'kibana' {
      name    = 'kibana'
      relNotes = 'A browser-based analytics and search dashboard'
      version { base = '5.4.1'; pkg = base; release = 1 }
      tarball { destination = "$name-${version.base}.tar.gz"
                source      = "v${version.base}.zip" }
      url     { site = "https://github.com/elastic/kibana/archive"
                archive = site }
    }
  }
}