mirrors/bitcoin-s
1
0
Fork 0
mirror of https://github.com/bitcoin-s/bitcoin-s.git synced 2025-02-22 06:31:55 +01:00
Code Issues Projects Releases Packages Wiki Activity

Refactor zipDatadir (#3999)

Browse source 
* Refactor zipDatadir

* cleanup
This commit is contained in:
rorp 2022-01-21 08:03:53 -08:00 • committed by GitHub
parent 13f5fb8dcb
commit f438ce2897
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 163 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/oracle-server/src/main/scala/org/bitcoins/oracle/server/OracleServerMain.scala
View file

@ -22,7 +22,7 @@ class OracleServerMain(override val serverArgParser: ServerArgParser)(implicit
case None => conf.rpcBindOpt
}
val commonRoutes = CommonRoutes()
val commonRoutes = CommonRoutes(conf.baseDatadir)
for {
_ <- conf.start()

3
app/scripts/src/main/scala/org/bitcoins/scripts/ZipDatadir.scala
View file

@ -2,6 +2,7 @@ package org.bitcoins.scripts
import akka.actor.ActorSystem
import org.bitcoins.commons.util.{DatadirParser, ServerArgParser}
import org.bitcoins.db.DatadirUtil
import org.bitcoins.server.BitcoinSAppConfig
import org.bitcoins.server.routes.BitcoinSServerRunner
import org.bitcoins.server.util.BitcoinSAppScalaDaemon
@ -19,7 +20,7 @@ class ZipDatadir(override val serverArgParser: ServerArgParser)(implicit
//replace the line below with where you want to zip too
val path = Paths.get("/tmp", "bitcoin-s.zip")
val target = conf.zipDatadir(path)
val target = DatadirUtil.zipDatadir(conf.datadir, path)
logger.info(s"Done zipping to $target!")
for {
_ <- system.terminate()

48
app/server-routes/src/main/scala/org/bitcoins/server/routes/CommonRoutes.scala
View file

@ -1,17 +1,57 @@
package org.bitcoins.server.routes
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.server.directives.RouteDirectives
import org.bitcoins.core.util.EnvUtil
import org.bitcoins.db.DatadirUtil
case class CommonRoutes() extends ServerRoute {
import java.io.File
import java.nio.file.Path
import scala.util.{Failure, Success, Try}
case class CommonRoutes(datadir: Path) extends ServerRoute {
override def handleCommand: PartialFunction[ServerCommand, Route] = {
case ServerCommand("getversion", _) =>
RouteDirectives.complete {
val vec = Vector(("version", ujson.Str(EnvUtil.getVersion)))
complete {
val version =
Option(EnvUtil.getVersion).map(ujson.Str).getOrElse(ujson.Null)
val vec = Vector(("version", version))
val obj = ujson.Obj.from(vec)
Server.httpSuccess(obj)
}
case ServerCommand("zipdatadir", arr) =>
withValidServerCommand(ZipDataDir.fromJsArr(arr)) {
case ZipDataDir(path) =>
complete {
DatadirUtil.zipDatadir(datadir, path) match {
case Success(_) => Server.httpSuccess(ujson.Null)
case Failure(ex) => Server.httpError(ex.getMessage)
}
}
}
}
}
case class ZipDataDir(path: Path)
object ZipDataDir {
def fromJsArr(jsArr: ujson.Arr): Try[ZipDataDir] = {
jsArr.arr.toList match {
case pathJs :: Nil =>
Try {
val path = new File(pathJs.str).toPath
ZipDataDir(path)
}
case Nil =>
Failure(new IllegalArgumentException("Missing path argument"))
case other =>
Failure(
new IllegalArgumentException(
s"Bad number of arguments: ${other.length}. Expected: 1"))
}
}
}

35
app/server-test/src/test/scala/org/bitcoins/server/BitcoinSAppConfigTest.scala
View file

@ -1,35 +0,0 @@
package org.bitcoins.server
import org.bitcoins.testkit.fixtures.BitcoinSAppConfigBitcoinFixtureNotStarted
import org.bitcoins.testkit.util.FileUtil
import java.nio.file.Files
import scala.concurrent.Future
class BitcoinSAppConfigTest extends BitcoinSAppConfigBitcoinFixtureNotStarted {
behavior of "BitcoinSAppConfig"
it must "zipdatadir if the target directory is not created" in { config =>
val startF = config.start()
val fileName = FileUtil.randomDirName
val dir = Files
.createTempDirectory("hello")
//delete it
assert(Files.deleteIfExists(dir))
val target = dir.resolve(fileName)
assert(!Files.exists(target))
assert(!Files.exists(dir))
for {
_ <- startF
_ <- Future.fromTry(config.zipDatadir(target))
} yield {
assert(Files.exists(target))
}
}
}

50
app/server-test/src/test/scala/org/bitcoins/server/CommonRoutesSpec.scala
View file

@ -1,27 +1,67 @@
package org.bitcoins.server
import akka.http.scaladsl.model.ContentTypes
import akka.http.scaladsl.testkit.ScalatestRouteTest
import org.bitcoins.core.util.EnvUtil
import org.bitcoins.server.routes.{CommonRoutes, ServerCommand}
import org.bitcoins.testkit.BitcoinSTestAppConfig
import org.bitcoins.testkit.util.FileUtil
import org.bitcoins.testkit.util.FileUtil.withTempDir
import org.scalamock.scalatest.MockFactory
import org.scalatest.wordspec.AnyWordSpec
import java.nio.file.Files
class CommonRoutesSpec
extends AnyWordSpec
with ScalatestRouteTest
with MockFactory {
val commonRoutes = CommonRoutes()
implicit val conf: BitcoinSAppConfig =
BitcoinSTestAppConfig.getSpvTestConfig()
val commonRoutes = CommonRoutes(conf.datadir)
"CommonRoutes" should {
"getversion" in {
val version =
Option(EnvUtil.getVersion).map(v => "\"" + v + "\"").getOrElse("null")
val expectedJson =
ujson.read(s"""{"result":{"version":$version},"error":null}""")
val route =
commonRoutes.handleCommand(ServerCommand("getversion", ujson.Arr()))
Get() ~> route ~> check {
s"""
|{ "version" : "${EnvUtil.getVersion}" }
|""".stripMargin
Post() ~> route ~> check {
assert(contentType == ContentTypes.`application/json`)
val actualJson = ujson.read(responseAs[String])
assert(actualJson == expectedJson)
}
}
"zipdatadir" in {
withTempDir(getClass.getName) { dir =>
val expectedJson =
ujson.read(s"""{"result":null,"error":null}""")
val fileName = FileUtil.randomDirName
val dirName = FileUtil.randomDirName
val target = dir.resolve(dirName).resolve(fileName)
assert(!Files.exists(target))
assert(!Files.exists(target.getParent))
val route =
commonRoutes.handleCommand(
ServerCommand("zipdatadir", ujson.Arr(target.toString)))
Post() ~> route ~> check {
assert(contentType == ContentTypes.`application/json`)
val actualJson = ujson.read(responseAs[String])
assert(actualJson == expectedJson)
assert(Files.exists(target))
}
}
}
}

2
app/server-test/src/test/scala/org/bitcoins/server/RoutesSpec.scala
View file

@ -82,7 +82,7 @@ class RoutesSpec extends AnyWordSpec with ScalatestRouteTest with MockFactory {
val coreRoutes: CoreRoutes = CoreRoutes()
val commonRoutes: CommonRoutes = CommonRoutes()
val commonRoutes: CommonRoutes = CommonRoutes(conf.datadir)
"The server" should {
"combine PSBTs" in {

51
app/server/src/main/scala/org/bitcoins/server/BitcoinSAppConfig.scala
View file

@ -5,11 +5,9 @@ import com.typesafe.config.{Config, ConfigFactory}
import grizzled.slf4j.Logging
import org.bitcoins.chain.config.ChainAppConfig
import org.bitcoins.commons.config.{AppConfig, ConfigOps}
import org.bitcoins.commons.file.FileUtil
import org.bitcoins.commons.util.ServerArgParser
import org.bitcoins.core.config.NetworkParameters
import org.bitcoins.core.util.{StartStopAsync, TimeUtil}
import org.bitcoins.db.SQLiteUtil
import org.bitcoins.dlc.node.config.DLCNodeAppConfig
import org.bitcoins.dlc.wallet.DLCAppConfig
import org.bitcoins.keymanager.config.KeyManagerAppConfig
@ -18,12 +16,10 @@ import org.bitcoins.rpc.config.BitcoindRpcAppConfig
import org.bitcoins.tor.config.TorAppConfig
import org.bitcoins.wallet.config.WalletAppConfig
import java.io.IOException
import java.nio.file.{Files, Path, Paths}
import java.util.concurrent.TimeUnit
import scala.concurrent.Future
import scala.concurrent.duration.{DurationInt, FiniteDuration}
import scala.util.Try
/** A unified config class for all submodules of Bitcoin-S
* that accepts configuration. Thanks to implicit definitions
@ -62,6 +58,8 @@ case class BitcoinSAppConfig(
lazy val network: NetworkParameters = chainConf.network
lazy val datadir: Path = directory
/** Initializes the wallet, node and chain projects */
override def start(): Future[Unit] = {
val start = TimeUtil.currentEpochMs
@ -156,11 +154,6 @@ case class BitcoinSAppConfig(
def withOverrides(configs: Config*): BitcoinSAppConfig = {
BitcoinSAppConfig(directory, configs ++ confs: _*)
}
/** Zips $HOME/.bitcoin-s
*/
def zipDatadir(target: Path): Try[Path] =
BitcoinSAppConfig.zipDatadir(directory, target)
}
/** Implicit conversions that allow a unified configuration
@ -262,44 +255,4 @@ object BitcoinSAppConfig extends Logging {
def toBitcoindRpcConf(conf: BitcoinSAppConfig): BitcoindRpcAppConfig = {
conf.bitcoindRpcConf
}
def zipDatadir(source: Path, target: Path): Try[Path] = Try {
if (Files.exists(target)) {
throw new IOException(
s"Cannot zip datadir. Target file already exists: $target")
}
val temp = Files.createTempDirectory(source, "backup")
try {
// we don't want to store chaindb.sqlite as these databases are huge
// skip logs and binaries as these can be large as well
val tempRE = (".*/" + temp.getFileName + "/.*").r
FileUtil.copyDirectory(
source = source,
target = temp,
fileNameFilter = Vector(".*.sqlite$".r,
".*.sqlite-shm$".r,
".*.sqlite-wal$".r,
".*bitcoin-s.log$".r,
".*/seeds/.*".r,
".*/tor/.*".r,
".*/binaries/.*".r,
".*.zip$".r,
tempRE)
)
SQLiteUtil.backupDirectory(source = source,
target = temp,
fileNameFilter =
Vector(".*chaindb.sqlite$".r, tempRE))
FileUtil.zipDirectory(
source = temp,
target = target
)
} finally {
FileUtil.removeDirectory(temp)
()
}
}
}

2
app/server/src/main/scala/org/bitcoins/server/BitcoinSServerMain.scala
View file

@ -386,7 +386,7 @@ class BitcoinSServerMain(override val serverArgParser: ServerArgParser)(implicit
val chainRoutes = ChainRoutes(chainApi, nodeConf.network)
val coreRoutes = CoreRoutes()
val dlcRoutes = DLCRoutes(dlcNode)
val commonRoutes = CommonRoutes()
val commonRoutes = CommonRoutes(conf.datadir)
val handlers =
Seq(walletRoutes,

12
app/server/src/main/scala/org/bitcoins/server/CoreRoutes.scala
View file

@ -19,7 +19,6 @@ import upickle.default._
import scala.collection.mutable
import scala.concurrent.Future
import scala.util.{Failure, Success}
case class CoreRoutes()(implicit system: ActorSystem, config: BitcoinSAppConfig)
extends ServerRoute {
@ -221,17 +220,6 @@ case class CoreRoutes()(implicit system: ActorSystem, config: BitcoinSAppConfig)
Server.httpSuccess(json)
}
}
case ServerCommand("zipdatadir", arr) =>
withValidServerCommand(ZipDataDir.fromJsArr(arr)) {
case ZipDataDir(path) =>
complete {
config.zipDatadir(path) match {
case Success(_) => Server.httpSuccess(ujson.Null)
case Failure(ex) => Server.httpError(ex.getMessage)
}
}
}
}
def combinePSBTs(psbts: Vector[PSBT]): Future[PSBT] = {

22
app/server/src/main/scala/org/bitcoins/server/ServerJsonModels.scala
View file

@ -1226,28 +1226,6 @@ object BumpFee extends ServerJsonModels {
}
}
case class ZipDataDir(path: Path)
object ZipDataDir extends ServerJsonModels {
def fromJsArr(jsArr: ujson.Arr): Try[ZipDataDir] = {
jsArr.arr.toList match {
case pathJs :: Nil =>
Try {
val path = new File(pathJs.str).toPath
ZipDataDir(path)
}
case Nil =>
Failure(new IllegalArgumentException("Missing path argument"))
case other =>
Failure(
new IllegalArgumentException(
s"Bad number of arguments: ${other.length}. Expected: 1"))
}
}
}
case class CreateContractInfo(
announcementTLV: OracleAnnouncementTLV,
totalCollateral: Satoshis,

2
bitcoind-rpc/src/main/scala/org/bitcoins/rpc/config/BitcoindRpcAppConfig.scala
View file

@ -35,7 +35,7 @@ case class BitcoindRpcAppConfig(
configs: Seq[Config]): BitcoindRpcAppConfig =
BitcoindRpcAppConfig(directory, configs: _*)
protected[bitcoins] def baseDatadir: Path = directory
override protected[bitcoins] def baseDatadir: Path = directory
override def start(): Future[Unit] = Future.unit

14
db-commons-test/src/test/scala/org/bitcoins/db/DBConfigTest.scala
View file

@ -2,12 +2,11 @@ package org.bitcoins.db
import com.typesafe.config.ConfigFactory
import org.bitcoins.chain.config.ChainAppConfig
import org.bitcoins.commons.file.FileUtil
import org.bitcoins.core.config.MainNet
import org.bitcoins.node.config.NodeAppConfig
import org.bitcoins.testkit.BitcoinSTestAppConfig
import org.bitcoins.testkit.BitcoinSTestAppConfig.ProjectType
import org.bitcoins.testkit.util.BitcoinSAsyncTest
import org.bitcoins.testkit.util.{BitcoinSAsyncTest, FileUtil}
import org.bitcoins.wallet.config.WalletAppConfig
import java.io.File
@ -95,13 +94,6 @@ class DBConfigTest extends BitcoinSAsyncTest {
assert(mainNetChainAppConfig.network == MainNet)
}
def withTempDir[T](f: Path => T): T = {
val dir = Files.createTempDirectory(getClass.getName)
try {
f(dir)
} finally {
FileUtil.removeDirectory(dir)
()
}
}
def withTempDir[T](f: Path => T): T =
FileUtil.withTempDir(getClass.getName)(f)
}

51
db-commons/src/main/scala/org/bitcoins/db/DatadirUtil.scala Normal file
View file

@ -0,0 +1,51 @@
package org.bitcoins.db
import org.bitcoins.commons.file.FileUtil
import java.io.IOException
import java.nio.file.{Files, Path}
import scala.util.Try
object DatadirUtil {
def zipDatadir(source: Path, target: Path): Try[Path] = Try {
if (Files.exists(target)) {
throw new IOException(
s"Cannot zip datadir. Target file already exists: $target")
}
val temp = Files.createTempDirectory(source, "backup")
try {
// we don't want to store chaindb.sqlite as these databases are huge
// skip logs and binaries as these can be large as well
val tempRE = (".*/" + temp.getFileName + "/.*").r
FileUtil.copyDirectory(
source = source,
target = temp,
fileNameFilter = Vector(".*.sqlite$".r,
".*.sqlite-shm$".r,
".*.sqlite-wal$".r,
".*bitcoin-s.log$".r,
".*/seeds/.*".r,
".*/tor/.*".r,
".*/binaries/.*".r,
".*.zip$".r,
tempRE)
)
SQLiteUtil.backupDirectory(source = source,
target = temp,
fileNameFilter =
Vector(".*chaindb.sqlite$".r, tempRE))
FileUtil.zipDirectory(
source = temp,
target = target
)
} finally {
FileUtil.removeDirectory(temp)
()
}
}
}

13
testkit/src/main/scala/org/bitcoins/testkit/util/FileUtil.scala
View file

@ -3,7 +3,7 @@ package org.bitcoins.testkit.util
import grizzled.slf4j.Logging
import java.io.File
import java.nio.file.{Path, Paths}
import java.nio.file.{Files, Path, Paths}
import scala.annotation.tailrec
import scala.util.{Properties, Random}
@ -60,4 +60,15 @@ object FileUtil extends Logging {
f.mkdirs()
f
}
def withTempDir[T](prefix: String)(f: Path => T): T = {
val dir = Files.createTempDirectory(prefix)
try {
f(dir)
} finally {
deleteTmpDir(dir)
()
}
}
}