mirror of
https://github.com/mii443/wasmer.git
synced 2025-08-23 00:45:32 +00:00
383 lines
13 KiB
Python
383 lines
13 KiB
Python
#! /usr/bin/env python3
|
|
|
|
"""This is a script for publishing the wasmer crates to crates.io.
|
|
It should be run in the root of wasmer like `python3 scripts/publish.py --help`.
|
|
|
|
Please lint with pylint and format with black.
|
|
|
|
$ pylint scripts/publish.py
|
|
|
|
--------------------------------------------------------------------
|
|
Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00)
|
|
|
|
$ black scripts/publish.py
|
|
"""
|
|
|
|
import argparse
|
|
import itertools
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import sys
|
|
import typing
|
|
from pprint import pprint
|
|
|
|
try:
|
|
# try import tomllib from standard Python library: New in version 3.11.
|
|
import tomllib
|
|
except ModuleNotFoundError:
|
|
try:
|
|
# if tomllib is missing, this is an older python3 version. Try
|
|
# importing the equivalent third-party library tomli:
|
|
import tomli as tomllib
|
|
except ModuleNotFoundError:
|
|
print("Please install tomli, `pip3 install tomli`")
|
|
sys.exit(1)
|
|
|
|
try:
|
|
from graphlib import TopologicalSorter
|
|
except ImportError:
|
|
print("Incompatible python3 version: lacks graphlib standard library module.")
|
|
sys.exit(1)
|
|
|
|
SETTINGS = {
|
|
# extra features to put when publishing, for example wasmer-cli needs a
|
|
# compiler by default otherwise it won't work standalone
|
|
"publish_features": {
|
|
"wasmer-cli": "default,cranelift",
|
|
"wasmer-wasix": "sys,wasmer/sys",
|
|
"wasmer-wasix-types": "wasmer/sys",
|
|
"wasmer-wast": "wasmer/sys",
|
|
"wai-bindgen-wasmer": "sys",
|
|
"wasmer-cache": "wasmer/sys",
|
|
"wasmer-emscripten": "wasmer/sys"
|
|
},
|
|
# workspace members we want to publish but whose path doesn't start by
|
|
# "./lib/"
|
|
"non-lib-workspace-members": set(["tests/lib/wast"]),
|
|
}
|
|
|
|
|
|
def get_latest_version_for_crate(crate_name: str) -> typing.Optional[str]:
|
|
"""Fetches the latest version of a given crate name in local cargo registry."""
|
|
output = subprocess.run(
|
|
["cargo", "search", crate_name], capture_output=True, check=True
|
|
)
|
|
rexp_src = f'^{crate_name} = "([^"]+)"'
|
|
prog = re.compile(rexp_src)
|
|
haystack = output.stdout.decode("utf-8")
|
|
for line in haystack.splitlines():
|
|
result = prog.match(line)
|
|
if result:
|
|
return result.group(1)
|
|
return None
|
|
|
|
|
|
class Crate:
|
|
"""Represents a workspace crate that is to be published to crates.io."""
|
|
|
|
def __init__(
|
|
self,
|
|
dependencies: typing.List[str],
|
|
cargo_manifest: dict,
|
|
cargo_file_path="Cargo.toml",
|
|
):
|
|
self.name = cargo_manifest["package"]["name"]
|
|
self.dependencies = dependencies
|
|
self.cargo_manifest = cargo_manifest
|
|
if not os.path.isabs(cargo_file_path):
|
|
cargo_file_path = os.path.join(os.getcwd(), cargo_file_path)
|
|
self.cargo_file_path = cargo_file_path
|
|
|
|
def __str__(self):
|
|
return f"{self.name}: {self.dependencies} {self.cargo_file_path} {self.path()}"
|
|
|
|
@property
|
|
def path(self) -> str:
|
|
"""Return the absolute filesystem path containing this crate."""
|
|
return os.path.dirname(self.cargo_file_path)
|
|
|
|
@property
|
|
def version(self) -> str:
|
|
"""Return the crate's version according to its manifest."""
|
|
return self.cargo_manifest["package"]["version"]
|
|
|
|
|
|
class Publisher:
|
|
"""A class responsible for figuring out dependencies,
|
|
creating a topological sorting in order to publish them
|
|
to crates.io in a valid order."""
|
|
|
|
def __init__(self, version=None, dry_run=True, verbose=True):
|
|
self.dry_run: bool = dry_run
|
|
self.verbose: bool = verbose
|
|
|
|
# open workspace Cargo.toml
|
|
with open("Cargo.toml", "rb") as file:
|
|
data = tomllib.load(file)
|
|
|
|
if version is None:
|
|
version = data["workspace"]["package"]["version"]
|
|
self.version: str = version
|
|
|
|
if self.verbose and not self.dry_run:
|
|
print(f"Publishing version {self.version}")
|
|
elif self.verbose and self.dry_run:
|
|
print(f"Publishing version {self.version} dry run!")
|
|
|
|
# define helper function
|
|
check_local_dep_fn = lambda t: isinstance(t[1], dict) and "path" in t[1]
|
|
members = set(
|
|
map(
|
|
lambda p: p + "/Cargo.toml",
|
|
filter(
|
|
lambda path: (
|
|
path.startswith("lib/") and os.path.exists(path + "/Cargo.toml")
|
|
)
|
|
or path in SETTINGS["non-lib-workspace-members"],
|
|
itertools.chain(
|
|
data["workspace"]["members"],
|
|
map(
|
|
lambda p: p[1]["path"],
|
|
filter(check_local_dep_fn, data["dependencies"].items()),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
)
|
|
crates = []
|
|
for member in members:
|
|
with open(member, "rb") as file:
|
|
member_data = tomllib.load(file)
|
|
|
|
def return_dependencies(toml) -> typing.List[str]:
|
|
acc = set()
|
|
stack = [toml]
|
|
while len(stack) > 0:
|
|
toml = stack.pop()
|
|
if "dependencies" in toml:
|
|
acc.update(
|
|
list(
|
|
map(
|
|
lambda dep: dep[1]["package"]
|
|
if "package" in dep[1]
|
|
else dep[0],
|
|
filter(
|
|
check_local_dep_fn, toml["dependencies"].items()
|
|
),
|
|
)
|
|
)
|
|
)
|
|
if "dev-dependencies" in toml:
|
|
acc.update(
|
|
list(
|
|
map(
|
|
lambda dep: dep[1]["package"]
|
|
if "package" in dep[1]
|
|
else dep[0],
|
|
filter(
|
|
check_local_dep_fn, toml["dev-dependencies"].items()
|
|
),
|
|
)
|
|
)
|
|
)
|
|
if "target" in toml:
|
|
stack.append(toml["target"])
|
|
for key, value in toml.items():
|
|
if key.startswith("cfg"):
|
|
stack.append(value)
|
|
return list(acc)
|
|
|
|
dependencies = return_dependencies(member_data)
|
|
crates.append(Crate(dependencies, member_data, cargo_file_path=member))
|
|
|
|
self.crates = crates
|
|
self.crate_index: typing.Dict[str, Crate] = {c.name: c for c in crates}
|
|
|
|
self.create_publish_order()
|
|
|
|
self.starting_dir = os.getcwd()
|
|
|
|
def create_publish_order(self):
|
|
"""Creates a valid publish order by topologically
|
|
sorting crates using the dependency graph."""
|
|
topological_sorter = TopologicalSorter()
|
|
|
|
for crate in self.crates:
|
|
topological_sorter.add(crate.name, *crate.dependencies)
|
|
|
|
self.publish_order: typing.List[str] = [*topological_sorter.static_order()]
|
|
|
|
def is_crate_already_published(self, crate_name: str) -> bool:
|
|
"""Checks if a given crate name is already published
|
|
with the version string we intend to publish with."""
|
|
found_string: str = get_latest_version_for_crate(crate_name)
|
|
if found_string is None:
|
|
return False
|
|
|
|
if self.version == found_string:
|
|
return True
|
|
|
|
crate = self.crate_index[crate_name]
|
|
with open(crate.path + "/Cargo.toml", "rb") as file:
|
|
data = tomllib.load(file)
|
|
crate_version = data["package"]["version"]
|
|
|
|
if crate_version is None:
|
|
return False
|
|
|
|
return crate_version == found_string
|
|
|
|
def publish_crate(self, crate_name: str):
|
|
# pylint: disable=broad-except
|
|
"""Publish a given crate by name."""
|
|
status = None
|
|
try:
|
|
crate = self.crate_index[crate_name]
|
|
os.chdir(crate.path)
|
|
extra_args = []
|
|
if crate_name in SETTINGS["publish_features"]:
|
|
extra_args = ["--features", SETTINGS["publish_features"][crate_name]]
|
|
if self.dry_run:
|
|
print(f"In dry-run: not publishing crate `{crate_name}`")
|
|
command = ["cargo", "publish", "--dry-run"] + extra_args
|
|
if self.verbose:
|
|
print(*command)
|
|
output = subprocess.run(command, check=True)
|
|
if self.verbose:
|
|
print(output)
|
|
else:
|
|
command = ["cargo", "publish"] + extra_args
|
|
if self.verbose:
|
|
print(*command)
|
|
output = subprocess.run(command, check=True)
|
|
if self.verbose:
|
|
print(output)
|
|
if self.verbose:
|
|
print("Success.")
|
|
except Exception as exc:
|
|
if self.verbose:
|
|
print(f"Failed to publish {crate_name}.")
|
|
print(exc)
|
|
status = exc
|
|
finally:
|
|
os.chdir(self.starting_dir)
|
|
return status
|
|
|
|
def publish(self):
|
|
# pylint: disable=too-many-branches
|
|
"""Publish all packages in workspace."""
|
|
if self.verbose and self.dry_run:
|
|
print("(Dry run, not actually publishing anything)")
|
|
if self.verbose:
|
|
print("Publishing order:")
|
|
pprint(self.publish_order)
|
|
status = {}
|
|
failures = 0
|
|
|
|
for crate_name in self.publish_order:
|
|
print(f"Publishing `{crate_name}`...")
|
|
if not self.is_crate_already_published(crate_name):
|
|
status[crate_name] = self.publish_crate(crate_name)
|
|
if status[crate_name]:
|
|
failures = +1
|
|
else:
|
|
print(f"`{crate_name}` was already published!")
|
|
continue
|
|
|
|
# sleep for 16 seconds between crates to ensure the crates.io
|
|
# index has time to update
|
|
if not self.dry_run:
|
|
print(
|
|
"Sleeping for 16 seconds to allow the `crates.io` index to update..."
|
|
)
|
|
time.sleep(16)
|
|
else:
|
|
print("In dry-run: not sleeping for crates.io to update.")
|
|
if failures > 0 and self.verbose:
|
|
print(f"encountered {failures} failures.")
|
|
for key, value in status.items():
|
|
if value is None:
|
|
result = "ok"
|
|
else:
|
|
result = str(value)
|
|
print(f"{key}\t{result}")
|
|
if self.verbose:
|
|
print(f"Published {len(status) - failures} crates.")
|
|
return failures
|
|
|
|
|
|
def main():
|
|
"""Main executable function."""
|
|
os.environ["WASMER_PUBLISH_SCRIPT_IS_RUNNING"] = "1"
|
|
parser = argparse.ArgumentParser(
|
|
description="Publish the Wasmer crates to crates.io"
|
|
)
|
|
subparsers = parser.add_subparsers(dest="subcommand")
|
|
health_cmd = subparsers.add_parser(
|
|
"health-check",
|
|
help="""Check the dependency graph is a tree, meaning a non-cyclic
|
|
planar graph. Combine with verbosity to print a topological sorting
|
|
of the graph.""",
|
|
)
|
|
health_cmd.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
default=False,
|
|
action="store_true",
|
|
help="Be verbose.",
|
|
)
|
|
health_cmd.add_argument(
|
|
"--print-dependencies",
|
|
default=False,
|
|
action="store_true",
|
|
help="For each crate, print its dependencies.",
|
|
)
|
|
publish_cmd = subparsers.add_parser(
|
|
"publish", help="Publish Wasmer crates to crates.io."
|
|
)
|
|
publish_cmd.add_argument(
|
|
"--version",
|
|
default=None,
|
|
type=str,
|
|
help="""Define the semver target triple (Default is automatically
|
|
read from workspace Cargo.toml.""",
|
|
)
|
|
publish_cmd.add_argument(
|
|
"--dry-run",
|
|
default=False,
|
|
action="store_true",
|
|
help="""Run the script without actually publishing anything to
|
|
crates.io""",
|
|
)
|
|
publish_cmd.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
default=False,
|
|
action="store_true",
|
|
help="Be verbose.",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
if args.subcommand == "health-check":
|
|
verbose = args.verbose
|
|
publisher = Publisher(verbose=verbose)
|
|
if verbose:
|
|
print(f"Version is {publisher.version}")
|
|
pprint(publisher.publish_order)
|
|
if args.print_dependencies:
|
|
for crate in publisher.crates:
|
|
print(f"{crate.name}: {crate.dependencies}")
|
|
return 0
|
|
|
|
if args.subcommand == "publish":
|
|
verbose = args.verbose
|
|
publisher = Publisher(dry_run=args.dry_run, verbose=verbose)
|
|
return publisher.publish()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|