2022 05 05 OS specific jlink builds (#4322)

* Add OS specific zips

* Try to fix windows build

* Try to fix windows build shell

* Try to see if double zip works w/o mac verified developer error

* Add bash script to handle making file executable and make it non quarantine on mac

* Do chmod regardless of os

* Remove chmod as that cannot work by definition, revert to only packaging stage/ directory

* Make java exectuable on every OS

* refactor CommonSettings.jlinkIgnore

* Try to build appServer

* Make cli use jlink

* Make bash scripts more generic on mac

* Remove feature branch on release.yml

* Fix startup scripts

.github/workflows/release.yml

@@ -194,7 +194,76 @@ jobs:
           files: "D:\\a\\bitcoin-s\\bitcoin-s\\app\\bundle\\target\\windows\\bitcoin-s-bundle.msi"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  zip:
+  mac-zips:
+    runs-on: [ macos-latest ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v13
+        with:
+          java-version: openjdk@1.17.0
+      - name: 'Get Previous tag'
+        id: previoustag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        with:
+          fallback: 1.0.0 # Optional fallback tag to use when no tag can be found
+      - name: Build zip
+        run: sbt "universal:stage;universal:packageBin"
+      - name: View Artifacts
+        run: ls -l app/server/target/universal/stage
+      - name: pwd
+        run: pwd
+      - name: Upload bitcoin-s-server-mac zip
+        uses: actions/upload-artifact@v3
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+        with:
+          name: "bitcoin-s-server-mac"
+          path: app/server/target/universal/stage
+      - name: Upload bitcoin-s-oracle-server zip
+        uses: actions/upload-artifact@v3
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+        with:
+          name: "bitcoin-s-oracle-server-mac"
+          path: app/oracle-server/target/universal/stage
+      - name: Upload bitcoin-s-cli zip
+        uses: actions/upload-artifact@v3
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+        with:
+          name: "bitcoin-s-cli-mac"
+          path: app/cli/target/universal/stage/
+      - name: Upload bitcoin-s-server if release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          name: "bitcoin-s-server-mac-${{steps.previoustag.outputs.tag}}"
+          files: app/server/target/universal/*.zip
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload bitcoin-s-oracle-server if release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          name: "bitcoin-s-oracle-server-mac-${{steps.previoustag.outputs.tag}}"
+          files: app/oracle-server/target/universal/*.zip
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload bitcoin-s-cli if release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          name: "bitcoin-s-cli-mac-${{steps.previoustag.outputs.tag}}"
+          files: app/cli/target/universal/*.zip
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  linux-zips:
     runs-on: [ ubuntu-latest ]
     steps:
       - uses: actions/checkout@v2
@@ -215,36 +284,32 @@ jobs:
         run: ls -l app/server/target/universal/stage
       - name: pwd
         run: pwd
-      - name: chmod startup script
-        run: |
-          chmod +x app/server/target/universal/stage/bin/bitcoin-s-server
-          chmod +x app/server/target/universal/stage/bin/bitcoin-s-server.bat
-      - name: Upload bitcoin-s-server zip
+      - name: Upload bitcoin-s-server-linux zip
         uses: actions/upload-artifact@v3
         env:
           pkg-version: ${{steps.previoustag.outputs.tag}}
         with:
-          name: "bitcoin-s-server"
+          name: "bitcoin-s-server-linux"
           path: app/server/target/universal/stage/
-      - name: Upload bitcoin-s-oracle-server zip
+      - name: Upload bitcoin-s-oracle-server-linux zip
         uses: actions/upload-artifact@v3
         env:
           pkg-version: ${{steps.previoustag.outputs.tag}}
         with:
-          name: "bitcoin-s-oracle-server"
+          name: "bitcoin-s-oracle-server-linux"
           path: app/oracle-server/target/universal/stage/
-      - name: Upload bitcoin-s-cli zip
+      - name: Upload bitcoin-s-cli-linux zip
         uses: actions/upload-artifact@v3
         env:
           pkg-version: ${{steps.previoustag.outputs.tag}}
         with:
-          name: "bitcoin-s-cli"
+          name: "bitcoin-s-cli-linux"
           path: app/cli/target/universal/stage/
       - name: Upload bitcoin-s-server if release
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')
         with:
-          name: "bitcoin-s-server-${{steps.previoustag.outputs.tag}}"
+          name: "bitcoin-s-server-linux-${{steps.previoustag.outputs.tag}}"
           files: app/server/target/universal/*.zip
         env:
           pkg-version: ${{steps.previoustag.outputs.tag}}
@@ -253,7 +318,7 @@ jobs:
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')
         with:
-          name: "bitcoin-s-oracle-server-${{steps.previoustag.outputs.tag}}"
+          name: "bitcoin-s-oracle-server-linux-${{steps.previoustag.outputs.tag}}"
           files: app/oracle-server/target/universal/*.zip
         env:
           pkg-version: ${{steps.previoustag.outputs.tag}}
@@ -262,7 +327,77 @@ jobs:
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')
         with:
-          name: "bitcoin-s-cli-${{steps.previoustag.outputs.tag}}"
+          name: "bitcoin-s-cli-linux-${{steps.previoustag.outputs.tag}}"
+          files: app/cli/target/universal/*.zip
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  windows-zips:
+    runs-on: [windows-latest]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v13
+        with:
+          java-version: openjdk@1.17.0
+      - name: 'Get Previous tag'
+        id: previoustag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        with:
+          fallback: 1.0.0 # Optional fallback tag to use when no tag can be found
+      - name: Build zip
+        shell: bash
+        run: sbt "universal:stage;universal:packageBin"
+      - name: View Artifacts
+        run: ls -l app/server/target/universal/stage
+      - name: pwd
+        run: pwd
+      - name: Upload bitcoin-s-server-windows zip
+        uses: actions/upload-artifact@v3
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+        with:
+          name: "bitcoin-s-server-windows"
+          path: app/server/target/universal/stage/
+      - name: Upload bitcoin-s-oracle-server zip
+        uses: actions/upload-artifact@v3
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+        with:
+          name: "bitcoin-s-oracle-server-windows"
+          path: app/oracle-server/target/universal/stage/
+      - name: Upload bitcoin-s-cli zip
+        uses: actions/upload-artifact@v3
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+        with:
+          name: "bitcoin-s-cli-windows"
+          path: app/cli/target/universal/stage/
+      - name: Upload bitcoin-s-server if release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          name: "bitcoin-s-server-windows-${{steps.previoustag.outputs.tag}}"
+          files: app/server/target/universal/*.zip
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload bitcoin-s-oracle-server if release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          name: "bitcoin-s-oracle-server-windows-${{steps.previoustag.outputs.tag}}"
+          files: app/oracle-server/target/universal/*.zip
+        env:
+          pkg-version: ${{steps.previoustag.outputs.tag}}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload bitcoin-s-cli if release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          name: "bitcoin-s-cli-windows-${{steps.previoustag.outputs.tag}}"
           files: app/cli/target/universal/*.zip
         env:
           pkg-version: ${{steps.previoustag.outputs.tag}}

app/cli/cli.sbt

@@ -9,5 +9,3 @@ nativeImageOptions ++= Seq(
   "--enable-http",
   "--enable-https"
 )
-
-enablePlugins(JavaAppPackaging, NativeImagePlugin)

app/cli/src/universal/cli-extra-startup-script.sh

@@ -0,0 +1,8 @@
+
+if [[ "$OS" == "OSX" ]]; then
+  #mac doesn't allow random binaries to be executable
+  #remove the quarantine attribute so java is executable on mac
+  xattr -d com.apple.quarantine jre/bin/java
+fi
+
+chmod +x jre/bin/java #make sure java is executable

app/oracle-server/src/universal/oracle-server-extra-startup-script.sh

@@ -0,0 +1,8 @@
+
+if [[ "$OS" == "OSX" ]]; then
+  #mac doesn't allow random binaries to be executable
+  #remove the quarantine attribute so java is executable on mac
+  xattr -d com.apple.quarantine jre/bin/java
+fi
+
+chmod +x jre/bin/java #make sure java is executable

app/server/src/universal/wallet-server-extra-startup-script.sh

@@ -0,0 +1,8 @@
+
+if [[ "$OS" == "OSX" ]]; then
+  #mac doesn't allow random binaries to be executable
+  #remove the quarantine attribute so java is executable on mac
+  xattr -d com.apple.quarantine jre/bin/java
+fi
+
+chmod +x jre/bin/java #make sure java is executable

build.sbt

@@ -374,6 +374,7 @@ lazy val oracleServer = project
   .settings(jlinkModules --= CommonSettings.rmJlinkModules)
   .settings(jlinkOptions ++= CommonSettings.jlinkOptions)
   .settings(jlinkIgnoreMissingDependency := CommonSettings.oracleServerJlinkIgnore)
+  .settings(bashScriptExtraDefines ++= IO.readLines(baseDirectory.value / "src" / "universal" / "oracle-server-extra-startup-script.sh"))
   .dependsOn(
     dlcOracle,
     serverRoutes
@@ -401,6 +402,11 @@ lazy val appServer = project
   .settings(CommonSettings.appSettings: _*)
   .settings(CommonSettings.dockerSettings: _*)
   .settings(CommonSettings.dockerBuildxSettings: _*)
+  .settings(jlinkModules ++= CommonSettings.jlinkModules)
+  .settings(jlinkModules --= CommonSettings.rmJlinkModules)
+  .settings(jlinkOptions ++= CommonSettings.jlinkOptions)
+  .settings(jlinkIgnoreMissingDependency := CommonSettings.appServerJlinkIgnore)
+  .settings(bashScriptExtraDefines ++= IO.readLines(baseDirectory.value / "src" / "universal" / "wallet-server-extra-startup-script.sh"))
   .dependsOn(
     serverRoutes,
     appCommons,
@@ -413,7 +419,7 @@ lazy val appServer = project
     feeProvider,
     zmq
   )
-  .enablePlugins(JavaAppPackaging, DockerPlugin)
+  .enablePlugins(JavaAppPackaging, DockerPlugin, JlinkPlugin)
 
 lazy val appServerTest = project
   .in(file("app/server-test"))
@@ -432,9 +438,13 @@ lazy val cli = project
   .settings(
     name := "bitcoin-s-cli"
   )
+  .settings(jlinkOptions ++= CommonSettings.jlinkOptions)
+  .settings(jlinkModules --= CommonSettings.rmCliJlinkModules)
+  .settings(jlinkIgnoreMissingDependency := CommonSettings.cliJlinkIgnore)
+  .settings(bashScriptExtraDefines ++= IO.readLines(baseDirectory.value / "src" / "universal" / "cli-extra-startup-script.sh"))
   .dependsOn(
     appCommons
-  )
+  ).enablePlugins(JavaAppPackaging, NativeImagePlugin, JlinkPlugin)
 
 lazy val cliTest = project
   .in(file("app/cli-test"))

project/CommonSettings.scala

@@ -106,6 +106,16 @@ object CommonSettings {
     "java.desktop"
   )
 
+  lazy val rmCliJlinkModules = {
+    rmJlinkModules ++ Vector(
+      "java.logging",
+      "java.naming",
+      "java.sql",
+      "java.xml",
+      "jdk.unsupported"
+    )
+  }
+
   lazy val jlinkOptions = Seq(
     "--no-header-files",
     "--no-man-pages",
@@ -252,59 +262,88 @@ object CommonSettings {
   lazy val binariesPath =
     Paths.get(Properties.userHome, ".bitcoin-s", "binaries")
 
-  lazy val oracleServerJlinkIgnore = {
-    JlinkIgnore.byPackagePrefix(
-      "java.xml" -> "java.activation",
+  lazy val dbCommonsJlinkIgnore = {
+    //we don't use android
+   Vector("org.flywaydb.core.api.android" -> "android.content",
+    "org.flywaydb.core.internal.logging.android" -> "android.util",
+    "org.flywaydb.core.internal.resource.android" -> "android.content.res",
+    "org.flywaydb.core.internal.scanner.android" -> "android.content",
+    "org.flywaydb.core.internal.scanner.android" -> "android.content.pm",
+    "org.flywaydb.core.internal.scanner.android" -> "android.content.res",
+    "org.flywaydb.core.internal.scanner.android" -> "dalvik.system",
+    //we don't use hibernate
+    "com.zaxxer.hikari.hibernate" -> "org.hibernate",
+
+    //we don't ship with support for any aws products
+    "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.awscore.exception",
+    "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.core",
+    "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.services.s3",
+    "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.services.s3.model",
+    "org.flywaydb.core.internal.scanner.cloud.s3" -> "software.amazon.awssdk.core.exception",
+    "org.flywaydb.core.internal.scanner.cloud.s3" -> "software.amazon.awssdk.services.s3",
+    "org.flywaydb.core.internal.scanner.cloud.s3" -> "software.amazon.awssdk.services.s3.model",
+    "org.flywaydb.core.api.configuration" -> "software.amazon.awssdk.services.s3",
+
+    //we don't use oracle database products
+    "org.flywaydb.core.internal.database.oracle" -> "oracle.jdbc",
+
+    //we don't use jboss
+    "org.flywaydb.core.internal.scanner.classpath.jboss" -> "org.jboss.vfs",
+
+    "org.flywaydb.core.internal.logging.log4j2" -> "org.apache.logging.log4j",
+
+    "com.zaxxer.hikari.metrics.micrometer" -> "io.micrometer.core.instrument",
+    "com.zaxxer.hikari.pool" -> "io.micrometer.core.instrument",
+
+    "slick.jdbc" -> "javax.xml.bind",
+    "com.zaxxer.hikari.metrics.prometheus" -> "io.prometheus.client",
+    "com.zaxxer.hikari.util" -> "javassist",
+    "com.zaxxer.hikari.util" -> "javassist.bytecode",
+
+      //postgres requires this weird waffle dep
+    "waffle.jaas" -> "java.security.acl"
+    )
+  }
+
+  lazy val loggingJlinkIgnore = {
+    Vector(
       "ch.qos.logback.core.net" -> "javax.mail",
       "ch.qos.logback.core.net" -> "javax.mail.internet",
-      "org.flywaydb.core.api.android" -> "android.content",
-      "org.flywaydb.core.internal.logging.android" -> "android.util",
+      "org.apache.log4j.jmx" -> "com.sun.jdmk.comm",
+    )
+  }
 
-      //we don't use android
-      "org.flywaydb.core.internal.resource.android" -> "android.content.res",
-      "org.flywaydb.core.internal.scanner.android" -> "android.content",
-      "org.flywaydb.core.internal.scanner.android" -> "android.content.pm",
-      "org.flywaydb.core.internal.scanner.android" -> "android.content.res",
-      "org.flywaydb.core.internal.scanner.android" -> "dalvik.system",
+  lazy val oracleServerJlinkIgnore = {
+    val oracleServerIgnore = Vector(
+      "java.xml" -> "java.activation",
+      "com.github.benmanes.caffeine" -> "javax.annotation",
+      "com.github.benmanes.caffeine.cache" -> "javax.annotation",
+      "com.github.benmanes.caffeine.cache.stats" -> "javax.annotation",
+      //optional
+      "org.codehaus.janino" -> "org.apache.tools.ant").++(loggingJlinkIgnore).++(dbCommonsJlinkIgnore)
+    JlinkIgnore.byPackagePrefix(oracleServerIgnore:_*)
+  }
 
-      //we don't use hibernate
-      "com.zaxxer.hikari.hibernate" -> "org.hibernate",
-
-      //we don't ship with support for any aws products
-      "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.awscore.exception",
-      "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.core",
-      "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.services.s3",
-      "org.flywaydb.core.internal.resource.s3" -> "software.amazon.awssdk.services.s3.model",
-      "org.flywaydb.core.internal.scanner.cloud.s3" -> "software.amazon.awssdk.core.exception",
-      "org.flywaydb.core.internal.scanner.cloud.s3" -> "software.amazon.awssdk.services.s3",
-      "org.flywaydb.core.internal.scanner.cloud.s3" -> "software.amazon.awssdk.services.s3.model",
-      "org.flywaydb.core.api.configuration" -> "software.amazon.awssdk.services.s3",
-
-      //we don't use oracle database products
-      "org.flywaydb.core.internal.database.oracle" -> "oracle.jdbc",
-
-      //we don't use jboss
-      "org.flywaydb.core.internal.scanner.classpath.jboss" -> "org.jboss.vfs",
-
-      "org.flywaydb.core.internal.logging.log4j2" -> "org.apache.logging.log4j",
-
-      "slick.jdbc" -> "javax.xml.bind",
+  lazy val appServerJlinkIgnore = {
 
+    val appServerIgnore = loggingJlinkIgnore.++(dbCommonsJlinkIgnore).++(Vector(
+      //https://github.com/janino-compiler/janino/blob/f6bb39d3137ad2e99b41ecc48aaaf8ab2644bd1c/janino/pom.xml#L37
+      "org.codehaus.janino" -> "org.apache.tools.ant",
       "com.github.benmanes.caffeine" -> "javax.annotation",
       "com.github.benmanes.caffeine.cache" -> "javax.annotation",
       "com.github.benmanes.caffeine.cache.stats" -> "javax.annotation",
 
-      "com.zaxxer.hikari.metrics.micrometer" -> "io.micrometer.core.instrument",
-      "com.zaxxer.hikari.pool" -> "io.micrometer.core.instrument",
+      "monix.execution.misc" -> "scala.tools.nsc"
+    ))
+    JlinkIgnore.byPackagePrefix(appServerIgnore:_*)
+  }
 
-      "waffle.jaas" -> "java.security.acl",
-
-      "org.apache.log4j.jmx" -> "com.sun.jdmk.comm",
-      //optional
-      "org.codehaus.janino" -> "org.apache.tools.ant",
-      "com.zaxxer.hikari.metrics.prometheus" -> "io.prometheus.client",
-      "com.zaxxer.hikari.util" -> "javassist",
-      "com.zaxxer.hikari.util" -> "javassist.bytecode"
+  lazy val cliJlinkIgnore = {
+    val cliIgnore = Vector(
+      "scala.meta.internal.svm_subs" -> "com.oracle.svm.core.annotate",
+      "org.slf4j" -> "org.slf4j.impl"
     )
+
+    JlinkIgnore.byPackagePrefix(cliIgnore:_*)
   }
 }

project/Deps.scala

@@ -455,11 +455,7 @@ object Deps {
     List(
       Compile.sttp,
       Compile.newMicroPickle.value,
-      Compile.scopt,
-      //we can remove this dependency when this is fixed
-      //https://github.com/oracle/graal/issues/1943
-      //see https://github.com/bitcoin-s/bitcoin-s/issues/1100
-      Compile.codehaus
+      Compile.scopt
     )
   }
 
@@ -470,18 +466,18 @@ object Deps {
                  Compile.zxingJ2SE) ++ Compile.javaFxDeps
 
   val server = Def.setting {
-    List(
+    Vector(
       Compile.newMicroPickle.value,
       Compile.logback,
       Compile.akkaActor,
       Compile.akkaHttp,
       Compile.akkaStream,
       Compile.akkaSlf4j
-    )
+    ) ++ appServerTransitiveDeps
   }
 
   /** Transitive dependencies needed for the oracleServer to build properly with jlink */
-  private val oracleServerTransitiveDeps = Vector(
+  private val serverTransitiveDeps = Vector(
     //transitive deps needed for jlink
     Compile.codehaus,
     Compile.gson,
@@ -507,6 +503,10 @@ object Deps {
     Compile.javaxJms
   )
 
+
+  private val appServerTransitiveDeps = serverTransitiveDeps
+  private val oracleServerTransitiveDeps = serverTransitiveDeps
+
   val oracleServer = Def.setting {
     Vector(
       Compile.newMicroPickle.value,