import hashlib import json import os import shutil import sys import urllib.request import zipfile from http import HTTPStatus from typing import List, NamedTuple, Optional import httpx from fastapi.exceptions import HTTPException from fastapi.responses import JSONResponse from loguru import logger from pydantic import BaseModel from starlette.types import ASGIApp, Receive, Scope, Send from lnbits.settings import settings class Extension(NamedTuple): code: str is_valid: bool is_admin_only: bool name: Optional[str] = None short_description: Optional[str] = None tile: Optional[str] = None contributors: Optional[List[str]] = None hidden: bool = False migration_module: Optional[str] = None db_name: Optional[str] = None hash: Optional[str] = "" @property def module_name(self): return ( f"lnbits.extensions.{self.code}" if self.hash == "" else f"lnbits.upgrades.{self.code}-{self.hash}.{self.code}" ) @classmethod def from_installable_ext(cls, ext_info: "InstallableExtension") -> "Extension": return Extension( code=ext_info.id, is_valid=True, is_admin_only=False, # todo: is admin only name=ext_info.name, hash=ext_info.hash if ext_info.module_installed else "", ) class ExtensionManager: def __init__(self, include_disabled_exts=False): self._disabled: List[str] = ( [] if include_disabled_exts else settings.lnbits_disabled_extensions ) self._admin_only: List[str] = settings.lnbits_admin_extensions self._extension_folders: List[str] = [ x[1] for x in os.walk(os.path.join(settings.lnbits_path, "extensions")) ][0] @property def extensions(self) -> List[Extension]: output: List[Extension] = [] if "all" in self._disabled: return output for extension in [ ext for ext in self._extension_folders if ext not in self._disabled ]: try: with open( os.path.join( settings.lnbits_path, "extensions", extension, "config.json" ) ) as json_file: config = json.load(json_file) is_valid = True is_admin_only = True if extension in self._admin_only else False except Exception: config = {} is_valid = False is_admin_only = False output.append( Extension( extension, is_valid, is_admin_only, config.get("name"), config.get("short_description"), config.get("tile"), config.get("contributors"), config.get("hidden") or False, config.get("migration_module"), config.get("db_name"), ) ) return output class ExtensionRelease(BaseModel): name: str version: str archive: str source_repo: str hash: Optional[str] published_at: Optional[str] html_url: Optional[str] description: Optional[str] @classmethod def from_github_release(cls, source_repo: str, r: dict) -> "ExtensionRelease": return ExtensionRelease( name=r["name"], description=r["name"], version=r["tag_name"], archive=r["zipball_url"], source_repo=source_repo, # description=r["body"], # bad for JSON published_at=r["published_at"], html_url=r["html_url"], ) @classmethod async def all_releases(cls, org, repo) -> List["ExtensionRelease"]: async with httpx.AsyncClient() as client: releases_url = f"https://api.github.com/repos/{org}/{repo}/releases" resp = await client.get(releases_url) if resp.status_code != 200: raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail=f"Cannot fetch extension releases: {releases_url}", ) releases = resp.json() return [ ExtensionRelease.from_github_release(f"{org}/{repo}", r) for r in releases ] class InstallableExtension(BaseModel): id: str name: str archive: str # todo: move to installed_release hash: str short_description: Optional[str] = None details: Optional[str] = None icon: Optional[str] = None icon_url: Optional[str] = None dependencies: List[str] = [] is_admin_only: bool = False version: str = "none" # todo: move to Release stars: int = 0 latest_release: Optional[ExtensionRelease] installed_release: Optional[ExtensionRelease] @property def zip_path(self) -> str: extensions_data_dir = os.path.join(settings.lnbits_data_folder, "extensions") os.makedirs(extensions_data_dir, exist_ok=True) return os.path.join(extensions_data_dir, f"{self.id}.zip") @property def ext_dir(self) -> str: return os.path.join("lnbits", "extensions", self.id) @property def ext_upgrade_dir(self) -> str: return os.path.join("lnbits", "upgrades", f"{self.id}-{self.hash}") @property def module_name(self) -> str: return f"lnbits.extensions.{self.id}" @property def module_installed(self) -> bool: return self.module_name in sys.modules def download_archive(self): ext_zip_file = self.zip_path if os.path.isfile(ext_zip_file): os.remove(ext_zip_file) try: download_url(self.archive, ext_zip_file) except Exception as ex: logger.warning(ex) raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail="Cannot fetch extension archive file", ) archive_hash = file_hash(ext_zip_file) if self.hash != archive_hash: # remove downloaded archive if os.path.isfile(ext_zip_file): os.remove(ext_zip_file) raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail="File hash missmatch. Will not install.", ) def extract_archive(self): shutil.rmtree(self.ext_dir, True) with zipfile.ZipFile(self.zip_path, "r") as zip_ref: zip_ref.extractall(os.path.join("lnbits", "extensions")) os.makedirs(os.path.join("lnbits", "upgrades"), exist_ok=True) shutil.rmtree(self.ext_upgrade_dir, True) with zipfile.ZipFile(self.zip_path, "r") as zip_ref: zip_ref.extractall(self.ext_upgrade_dir) def nofiy_upgrade(self) -> None: """Update the the list of upgraded extensions. The middleware will perform redirects based on this""" if not self.hash: return clean_upgraded_exts = list( filter( lambda old_ext: not old_ext.endswith(f"/{self.id}"), settings.lnbits_upgraded_extensions, ) ) settings.lnbits_upgraded_extensions = clean_upgraded_exts + [ f"{self.hash}/{self.id}" ] def clean_extension_files(self): # remove downloaded archive if os.path.isfile(self.zip_path): os.remove(self.zip_path) # remove module from extensions shutil.rmtree(self.ext_dir, True) shutil.rmtree(self.ext_upgrade_dir, True) @classmethod async def from_repo(cls, org, repository) -> Optional["InstallableExtension"]: try: repo, latest_release, config = await fetch_github_repo_info(org, repository) return InstallableExtension( id=repo["name"], name=config.get("name"), short_description=config.get("short_description"), archive="xx", hash="123", version="0", stars=repo["stargazers_count"], icon_url=icon_to_github_url(org, config.get("tile")), latest_release=ExtensionRelease.from_github_release( repo["html_url"], latest_release ), ) except Exception as e: logger.warning(e) return None @classmethod async def get_extension_info(cls, ext_id: str, hash: str) -> "InstallableExtension": installable_extensions: List[ InstallableExtension ] = await InstallableExtension.get_installable_extensions() valid_extensions = [ e for e in installable_extensions if e.id == ext_id and e.hash == hash ] if len(valid_extensions) == 0: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, detail=f"Unknown extension id: {ext_id}", ) extension = valid_extensions[0] # check that all dependecies are installed installed_extensions = list(map(lambda e: e.code, get_valid_extensions(True))) if not set(extension.dependencies).issubset(installed_extensions): raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail=f"Not all dependencies are installed: {extension.dependencies}", ) return extension @classmethod async def get_installable_extensions(cls) -> List["InstallableExtension"]: extension_list: List[InstallableExtension] = [] extension_id_list: List[str] = [] async with httpx.AsyncClient() as client: for url in settings.lnbits_extensions_manifests: try: resp = await client.get(url) if resp.status_code != 200: logger.warning(f"Cannot fetch extensions manifest at: {url}") continue manifest = resp.json() if "repos" in manifest: for r in manifest["repos"]: ext = await InstallableExtension.from_repo( r["organisation"], r["repository"] ) if ext: if ext.id in extension_id_list: continue extension_list += [ext] extension_id_list += [ext.id] if "extensions" in manifest: for e in manifest["extensions"]: if e["id"] in extension_id_list: continue extension_list += [ InstallableExtension( id=e["id"], name=e["name"], archive=e["archive"], hash=e["hash"], short_description=e["shortDescription"], details=e["details"] if "details" in e else "", icon=e["icon"], dependencies=e["dependencies"] if "dependencies" in e else [], ) ] extension_id_list += [e["id"]] except Exception as e: logger.warning(f"Manifest {url} failed with '{str(e)}'") return extension_list @classmethod async def get_extension_releases(cls, ext_id: str) -> List["ExtensionRelease"]: extension_releases: List[ExtensionRelease] = [] async with httpx.AsyncClient() as client: for url in settings.lnbits_extensions_manifests: try: resp = await client.get(url) if resp.status_code != 200: logger.warning(f"Cannot fetch extensions manifest at: {url}") continue manifest = resp.json() if "repos" in manifest: for r in manifest["repos"]: if r["id"] == ext_id: repo_releases = await ExtensionRelease.all_releases( r["organisation"], r["repository"] ) extension_releases += repo_releases if "extensions" in manifest: for e in manifest["extensions"]: if e["id"] == ext_id: extension_releases += [ ExtensionRelease( name=e["name"], version=e["version"], archive=e["archive"], hash=e["hash"], source_repo=url, description=e["shortDescription"], ) ] except Exception as e: logger.warning(f"Manifest {url} failed with '{str(e)}'") return extension_releases class InstalledExtensionMiddleware: def __init__(self, app: ASGIApp) -> None: self.app = app async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if not "path" in scope: await self.app(scope, receive, send) return path_elements = scope["path"].split("/") if len(path_elements) > 2: _, path_name, path_type, *rest = path_elements else: _, path_name = path_elements path_type = None # block path for all users if the extension is disabled if path_name in settings.lnbits_disabled_extensions: response = JSONResponse( status_code=HTTPStatus.NOT_FOUND, content={"detail": f"Extension '{path_name}' disabled"}, ) await response(scope, receive, send) return # re-route API trafic if the extension has been upgraded if path_type == "api": upgraded_extensions = list( filter( lambda ext: ext.endswith(f"/{path_name}"), settings.lnbits_upgraded_extensions, ) ) if len(upgraded_extensions) != 0: upgrade_path = upgraded_extensions[0] tail = "/".join(rest) scope["path"] = f"/upgrades/{upgrade_path}/{path_type}/{tail}" await self.app(scope, receive, send) def get_valid_extensions(include_disabled_exts=False) -> List[Extension]: return [ extension for extension in ExtensionManager(include_disabled_exts).extensions if extension.is_valid ] def download_url(url, save_path): with urllib.request.urlopen(url) as dl_file: with open(save_path, "wb") as out_file: out_file.write(dl_file.read()) def file_hash(filename): h = hashlib.sha256() b = bytearray(128 * 1024) mv = memoryview(b) with open(filename, "rb", buffering=0) as f: while n := f.readinto(mv): h.update(mv[:n]) return h.hexdigest() def icon_to_github_url(org: str, path: Optional[str]) -> str: if not path: return "" _, repo, *rest = path.split("/") tail = "/".join(rest) return f"https://github.com/{org}/{repo}/raw/main/{tail}" async def fetch_github_repo_info(org: str, repository: str): async with httpx.AsyncClient() as client: repo_url = f"https://api.github.com/repos/{org}/{repository}" resp = await client.get(repo_url) if resp.status_code != 200: raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail=f"Cannot fetch extension repo: {repo_url}", ) repo = resp.json() lates_release_url = ( f"https://api.github.com/repos/{org}/{repository}/releases/latest" ) resp = await client.get(lates_release_url) if resp.status_code != 200: raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail=f"Cannot fetch extension releases: {lates_release_url}", ) latest_release = resp.json() config_url = f"""https://raw.githubusercontent.com/{org}/{repository}/{repo["default_branch"]}/config.json""" resp = await client.get(config_url) if resp.status_code != 200: raise HTTPException( status_code=HTTPStatus.NOT_FOUND, detail=f"Cannot fetch config for extension: {config_url}", ) config = resp.json() return repo, latest_release, config