diff --git a/circup/backends.py b/circup/backends.py index 7af2295..8fc61d0 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -197,7 +197,9 @@ def install_module( # Create the library directory first. self.create_directory(device_path, library_path) if local_path is None: - if pyext: + # Fallback to the source version (py) if the bundle doesn't have + # a compiled version (mpy) + if pyext or bundle.platform is None: # Use Python source for module. self.install_module_py(metadata) else: @@ -648,9 +650,7 @@ def install_module_mpy(self, bundle, metadata): if not module_name: # Must be a directory based module. module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = self.get_circuitpython_version()[0].split(".")[0] - bundle_platform = "{}mpy".format(major_version) - bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) + bundle_path = os.path.join(bundle.lib_dir(), module_name) if os.path.isdir(bundle_path): self.install_dir_http(bundle_path) @@ -920,9 +920,7 @@ def install_module_mpy(self, bundle, metadata): # Must be a directory based module. module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = self.get_circuitpython_version()[0].split(".")[0] - bundle_platform = "{}mpy".format(major_version) - bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) + bundle_path = os.path.join(bundle.lib_dir(), module_name) if os.path.isdir(bundle_path): target_path = os.path.join(self.library_path, module_name) # Copy the directory. diff --git a/circup/bundle.py b/circup/bundle.py index 4a6dc7f..1d12481 100644 --- a/circup/bundle.py +++ b/circup/bundle.py @@ -20,7 +20,7 @@ from circup.logging import logger -class Bundle: +class Bundle: # pylint: disable=too-many-instance-attributes """ All the links and file names for a bundle """ @@ -50,29 +50,40 @@ def __init__(self, repo): self._latest = None self.pinned_tag = None self._available = [] + # + self.platform = None - def lib_dir(self, platform): + def lib_dir(self, source=False): """ - This bundle's lib directory for the platform. + This bundle's lib directory for the bundle's source or compiled version. - :param str platform: The platform identifier (py/6mpy/...). - :return: The path to the lib directory for the platform. + :param bool source: Whether to return the path to the source lib + directory or to :py:attr:`self.platform`'s lib directory. If `source` is + `False` but :py:attr:`self.platform` is None, the source lib directory + will be returned instead. + :return: The path to the lib directory. """ tag = self.current_tag + platform = "py" if source or not self.platform else self.platform return os.path.join( self.dir.format(platform=platform), self.basename.format(platform=PLATFORMS[platform], tag=tag), "lib", ) - def examples_dir(self, platform): + def examples_dir(self, source=False): """ - This bundle's examples directory for the platform. + This bundle's examples directory for the bundle's source or compiled + version. - :param str platform: The platform identifier (py/6mpy/...). - :return: The path to the examples directory for the platform. + :param bool source: Whether to return the path to the source examples + directory or to :py:attr:`self.platform`'s examples directory. If + `source` is `False` but :py:attr:`self.platform` is None, the source + examples directory will be returned instead. + :return: The path to the examples directory. """ tag = self.current_tag + platform = "py" if source or not self.platform else self.platform return os.path.join( self.dir.format(platform=platform), self.basename.format(platform=PLATFORMS[platform], tag=tag), @@ -104,18 +115,25 @@ def requirements_for(self, library_name, toml_file=False): def current_tag(self): """ The current tag for the project. If the tag hasn't been explicitly set - this will be the pinned tag, if one is set. If there is no pinned tag, - this will be the latest available tag that is locally available. + this will be the pinned tag, if one is set and it is available. If there + is no pinned tag, this will be the latest available tag that is locally + available. :return: The current tag value for the project. """ if self._current is None: - self._current = self.pinned_tag or ( - # This represents the latest version locally available - self._available[-1] - if len(self._available) > 0 - else None - ) + if self.pinned_tag: + self._current = ( + self.pinned_tag if self.pinned_tag in self._available else None + ) + else: + self._current = ( + # This represents the latest version locally available + self._available[-1] + if len(self._available) > 0 + else None + ) + return self._current @current_tag.setter @@ -161,6 +179,7 @@ def available_tags(self, tags): """ if isinstance(tags, str): tags = [tags] + # TODO: Need to pass int to sorted key...otherwise this might not sort them how it should self._available = sorted(tags) def add_tag(self, tag: str) -> None: @@ -187,22 +206,21 @@ def add_tag(self, tag: str) -> None: def validate(self): """ - Test the existence of the expected URLs (not their content) + Test the existence of the expected URL (not the content) """ tag = self.latest_tag if not tag or tag == "releases": if "--verbose" in sys.argv: click.secho(f' Invalid tag "{tag}"', fg="red") return False - for platform in PLATFORMS.values(): - url = self.url_format.format(platform=platform, tag=tag) - r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT) - # pylint: disable=no-member - if r.status_code != requests.codes.ok: - if "--verbose" in sys.argv: - click.secho(f" Unable to find {os.path.split(url)[1]}", fg="red") - return False - # pylint: enable=no-member + url = self.url_format.format(platform="py", tag=tag) + r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT) + # pylint: disable=no-member + if r.status_code != requests.codes.ok: + if "--verbose" in sys.argv: + click.secho(f" Unable to find {os.path.split(url)[1]}", fg="red") + return False + # pylint: enable=no-member return True def __repr__(self): diff --git a/circup/command_utils.py b/circup/command_utils.py index 6be718d..661b520 100644 --- a/circup/command_utils.py +++ b/circup/command_utils.py @@ -149,35 +149,67 @@ def ensure_bundle_tag(bundle, tag): logger.warning("Bundle version requested is 'None'.") return False - do_update = False + do_update_source = False + do_update_compiled = False if tag in bundle.available_tags: - for platform in PLATFORMS: - # missing directories (new platform added on an existing install - # or side effect of pytest or network errors) - do_update = do_update or not os.path.isdir(bundle.lib_dir(platform)) + # missing directories (new platform added on an existing install + # or side effect of pytest or network errors) + # Check for the source + do_update_source = not os.path.isdir(bundle.lib_dir(source=True)) + do_update_compiled = bundle.platform is not None and not os.path.isdir( + bundle.lib_dir(source=False) + ) else: - do_update = True + do_update_source = True + do_update_compiled = bundle.platform is not None + + if not (do_update_source or do_update_compiled): + logger.info("Current bundle version available (%s).", tag) + return True - if do_update: - if Bundle.offline: + if Bundle.offline: + if do_update_source: # pylint: disable=no-else-return logger.info( "Bundle version not available but skipping update in offline mode." ) return False + else: + logger.info( + "Bundle platform not available. Falling back to source (.py) files in offline mode." + ) + bundle.platform = None + return True - logger.info("New version available (%s).", tag) + logger.info("New version available (%s).", tag) + if do_update_source: try: - get_bundle(bundle, tag) - tags_data_save_tags(bundle.key, bundle.available_tags) + get_bundle(bundle, tag, "py") except requests.exceptions.HTTPError as ex: click.secho( - f"There was a problem downloading the {bundle.key} bundle.", fg="red" + f"There was a problem downloading the 'py' platform for the '{bundle.key}' bundle.", + fg="red", ) logger.exception(ex) - return False - else: - logger.info("Current bundle version available (%s).", tag) - return True + return False # Bundle isn't available + bundle.add_tag(tag) + tags_data_save_tags(bundle.key, bundle.available_tags) + + if do_update_compiled: + try: + get_bundle(bundle, tag, bundle.platform) + except requests.exceptions.HTTPError as ex: + click.secho( + ( + f"There was a problem downloading the '{bundle.platform}' platform for the " + f"'{bundle.key}' bundle.\nFalling back to source (.py) files." + ), + fg="red", + ) + logger.exception(ex) + bundle.platform = None # Compiled isn't available, source is good + bundle.current_tag = tag + + return True # bundle is available def ensure_latest_bundle(bundle): @@ -364,40 +396,39 @@ def find_modules(backend, bundles_list): # pylint: enable=broad-except,too-many-locals -def get_bundle(bundle, tag): +def get_bundle(bundle, tag, platform): """ Downloads and extracts the version of the bundle with the referenced tag. The resulting zip file is saved on the local filesystem. :param Bundle bundle: the target Bundle object. :param str tag: The GIT tag to use to download the bundle. + :param str platform: The platform string (i.e. '10mpy'). """ - click.echo(f"Downloading bundles for {bundle.key} ({tag}).") - for platform, github_string in PLATFORMS.items(): - # Report the platform: "8.x-mpy", etc. - click.echo(f"{github_string}:") - url = bundle.url_format.format(platform=github_string, tag=tag) - logger.info("Downloading bundle: %s", url) - r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT) - # pylint: disable=no-member - if r.status_code != requests.codes.ok: - logger.warning("Unable to connect to %s", url) - r.raise_for_status() - # pylint: enable=no-member - total_size = int(r.headers.get("Content-Length")) - temp_zip = bundle.zip.format(platform=platform) - with click.progressbar( - r.iter_content(1024), label="Extracting:", length=total_size - ) as pbar, open(temp_zip, "wb") as zip_fp: - for chunk in pbar: - zip_fp.write(chunk) - pbar.update(len(chunk)) - logger.info("Saved to %s", temp_zip) - temp_dir = bundle.dir.format(platform=platform) - with zipfile.ZipFile(temp_zip, "r") as zfile: - zfile.extractall(temp_dir) - bundle.add_tag(tag) - bundle.current_tag = tag + click.echo(f"Downloading '{platform}' bundle for {bundle.key} ({tag}).") + github_string = PLATFORMS[platform] + # Report the platform: "8.x-mpy", etc. + click.echo(f"{github_string}:") + url = bundle.url_format.format(platform=github_string, tag=tag) + logger.info("Downloading bundle: %s", url) + r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT) + # pylint: disable=no-member + if r.status_code != requests.codes.ok: + logger.warning("Unable to connect to %s", url) + r.raise_for_status() + # pylint: enable=no-member + total_size = int(r.headers.get("Content-Length")) + temp_zip = bundle.zip.format(platform=platform) + with click.progressbar( + r.iter_content(1024), label="Extracting:", length=total_size + ) as pbar, open(temp_zip, "wb") as zip_fp: + for chunk in pbar: + zip_fp.write(chunk) + pbar.update(len(chunk)) + logger.info("Saved to %s", temp_zip) + temp_dir = bundle.dir.format(platform=platform) + with zipfile.ZipFile(temp_zip, "r") as zfile: + zfile.extractall(temp_dir) click.echo("\nOK\n") @@ -417,9 +448,9 @@ def get_bundle_examples(bundles_list, avoid_download=False): try: for bundle in bundles_list: - if not avoid_download or not os.path.isdir(bundle.lib_dir("py")): + if not avoid_download or not os.path.isdir(bundle.lib_dir(source=True)): ensure_bundle(bundle) - path = bundle.examples_dir("py") + path = bundle.examples_dir(source=True) meta_saved = os.path.join(path, "../bundle_examples.json") if os.path.exists(meta_saved): with open(meta_saved, "r", encoding="utf-8") as f: @@ -465,9 +496,9 @@ def get_bundle_versions(bundles_list, avoid_download=False): """ all_the_modules = dict() for bundle in bundles_list: - if not avoid_download or not os.path.isdir(bundle.lib_dir("py")): + if not avoid_download or not os.path.isdir(bundle.lib_dir(source=True)): ensure_bundle(bundle) - path = bundle.lib_dir("py") + path = bundle.lib_dir(source=True) path_modules = _get_modules_file(path, logger) for name, module in path_modules.items(): module["bundle"] = bundle @@ -514,12 +545,14 @@ def get_bundles_local_dict(): return dict() -def get_bundles_list(bundle_tags): +def get_bundles_list(bundle_tags, platform_version=None): """ Retrieve the list of bundles from the config dictionary. :param Dict[str,str]|None bundle_tags: Pinned bundle tags. These override any tags found in the pyproject.toml. + :param str platform_version: The platform version needed for the current + device. :return: List of supported bundles as Bundle objects. """ bundle_config = get_bundles_dict() @@ -534,6 +567,7 @@ def get_bundles_list(bundle_tags): bundles_list = [Bundle(bundle_config[b]) for b in bundle_config] for bundle in bundles_list: + bundle.platform = platform_version bundle.available_tags = tags.get(bundle.key, []) if pinned_tags is not None: bundle.pinned_tag = pinned_tags.get(bundle.key) diff --git a/circup/commands.py b/circup/commands.py index 422f1fd..476c20e 100644 --- a/circup/commands.py +++ b/circup/commands.py @@ -215,6 +215,9 @@ def main( # pylint: disable=too-many-locals if board_id is None or cpy_version is None else (cpy_version, board_id) ) + major_version = cpy_version.split(".")[0] + bundle_platform = "{}mpy".format(major_version) + ctx.obj["DEVICE_PLATFORM_VERSION"] = bundle_platform click.echo( "Found device {} at {}, running CircuitPython {}.".format( board_id, device_path, cpy_version @@ -301,7 +304,10 @@ def list_cli(ctx): # pragma: no cover modules = [ m.row for m in find_modules( - ctx.obj["backend"], get_bundles_list(ctx.obj["BUNDLE_TAGS"]) + ctx.obj["backend"], + get_bundles_list( + ctx.obj["BUNDLE_TAGS"], ctx.obj["DEVICE_PLATFORM_VERSION"] + ), ) if m.outofdate ] @@ -378,7 +384,10 @@ def install( # pylint: disable=too-many-branches # TODO: Ensure there's enough space on the device - available_modules = get_bundle_versions(get_bundles_list(ctx.obj["BUNDLE_TAGS"])) + platform_version = ctx.obj["DEVICE_PLATFORM_VERSION"] if not pyext else None + available_modules = get_bundle_versions( + get_bundles_list(ctx.obj["BUNDLE_TAGS"], platform_version) + ) mod_names = {} for module, metadata in available_modules.items(): mod_names[module.replace(".py", "").lower()] = metadata @@ -600,7 +609,9 @@ def update(ctx, update_all): # pragma: no cover """ logger.info("Update") # Grab current modules. - bundles_list = get_bundles_list(ctx.obj["BUNDLE_TAGS"]) + bundles_list = get_bundles_list( + ctx.obj["BUNDLE_TAGS"], ctx.obj["DEVICE_PLATFORM_VERSION"] + ) installed_modules = find_modules(ctx.obj["backend"], bundles_list) modules_to_update = [m for m in installed_modules if m.outofdate] diff --git a/circup/module.py b/circup/module.py index 9336a2d..292a60a 100644 --- a/circup/module.py +++ b/circup/module.py @@ -79,16 +79,8 @@ def __init__( self.max_version = compatibility[1] # Figure out the bundle path. self.bundle_path = None - if self.mpy: - # Byte compiled, now check CircuitPython version. - - major_version = self.backend.get_circuitpython_version()[0].split(".")[0] - bundle_platform = "{}mpy".format(major_version) - else: - # Regular Python - bundle_platform = "py" # module path in the bundle - search_path = bundle.lib_dir(bundle_platform) + search_path = bundle.lib_dir(source=not self.mpy) if self.file: self.bundle_path = os.path.join(search_path, self.file) else: diff --git a/circup/shared.py b/circup/shared.py index cf0364b..a0339e1 100644 --- a/circup/shared.py +++ b/circup/shared.py @@ -22,7 +22,7 @@ DATA_DIR = appdirs.user_data_dir(appname="circup", appauthor="adafruit") #: Module formats list (and the other form used in github files) -PLATFORMS = {"py": "py", "9mpy": "9.x-mpy", "10mpy": "10.x-mpy"} +PLATFORMS = {"py": "py", "8mpy": "8.x-mpy", "9mpy": "9.x-mpy", "10mpy": "10.x-mpy"} #: Timeout for requests calls like get() REQUESTS_TIMEOUT = 30 diff --git a/tests/test_circup.py b/tests/test_circup.py index 491ace7..7e61375 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -51,7 +51,6 @@ pyproject_bundle_versions, tags_data_load, ) -from circup.shared import PLATFORMS from circup.module import Module from circup.logging import logger @@ -98,14 +97,15 @@ def test_Bundle_lib_dir(): Check the return of Bundle.lib_dir with a test tag. """ bundle = circup.Bundle(TEST_BUNDLE_NAME) + bundle.platform = "9mpy" with mock.patch.object(bundle, "_available", ["TESTTAG"]): assert bundle.current_tag == "TESTTAG" - assert bundle.lib_dir("py") == ( + assert bundle.lib_dir(source=True) == ( circup.shared.DATA_DIR + "/" "adafruit/adafruit-circuitpython-bundle-py/" "adafruit-circuitpython-bundle-py-TESTTAG/lib" ) - assert bundle.lib_dir("9mpy") == ( + assert bundle.lib_dir() == ( circup.shared.DATA_DIR + "/" "adafruit/adafruit-circuitpython-bundle-9mpy/" "adafruit-circuitpython-bundle-9.x-mpy-TESTTAG/lib" @@ -943,7 +943,7 @@ def test_ensure_latest_bundle_no_bundle_data(): ): bundle = circup.Bundle(TEST_BUNDLE_NAME) ensure_latest_bundle(bundle) - mock_gb.assert_called_once_with(bundle, "12345") + mock_gb.assert_called_once_with(bundle, "12345", "py") assert mock_json.dump.call_count == 1 # Current version saved to file. @@ -968,7 +968,7 @@ def test_ensure_latest_bundle_bad_bundle_data(): tags = tags_data_load() bundle.available_tags = tags.get(bundle.key, []) ensure_latest_bundle(bundle) - mock_gb.assert_called_once_with(bundle, "12345") + mock_gb.assert_called_once_with(bundle, "12345", "py") assert mock_logger.error.call_count == 1 assert mock_logger.exception.call_count == 1 @@ -987,7 +987,7 @@ def test_ensure_latest_bundle_to_update(): mock_json.load.return_value = {TEST_BUNDLE_NAME: "12345"} bundle = circup.Bundle(TEST_BUNDLE_NAME) ensure_latest_bundle(bundle) - mock_gb.assert_called_once_with(bundle, "54321") + mock_gb.assert_called_once_with(bundle, "54321", "py") assert mock_json.dump.call_count == 1 # Current version saved to file. @@ -1015,7 +1015,7 @@ def test_ensure_latest_bundle_to_update_http_error(): tags = tags_data_load() bundle.available_tags = tags.get(bundle.key, []) ensure_latest_bundle(bundle) - mock_gb.assert_called_once_with(bundle, "54321") + mock_gb.assert_called_once_with(bundle, "54321", "py") assert bundle.current_tag == "67890" assert mock_json.dump.call_count == 0 # not saved. assert mock_click.call_count == 2 # friendly message. @@ -1045,7 +1045,7 @@ def test_ensure_pinned_bundle_to_exit_http_error(): tags = tags_data_load() bundle.available_tags = tags.get(bundle.key, []) ensure_pinned_bundle(bundle) - mock_gb.assert_called_once_with(bundle, "54321") + mock_gb.assert_called_once_with(bundle, "54321", "py") assert mock_json.dump.call_count == 0 # not saved. assert mock_click.call_count == 2 # friendly message. mock_exit.assert_called_once_with(1) @@ -1074,6 +1074,34 @@ def test_ensure_latest_bundle_no_update(): assert mock_logger.info.call_count == 2 +def test_ensure_bundle_tag_fallback_to_source(): + """ + If a compiled platform download fails, fallback to the source version. + """ + tags_data = {TEST_BUNDLE_NAME: ["12345"]} + with mock.patch("circup.Bundle.latest_tag", "54321"), mock.patch( + "circup.os.path.isfile", + return_value=True, + ), mock.patch("circup.command_utils.open"), mock.patch( + "circup.command_utils.get_bundle", + side_effect=[None, requests.exceptions.HTTPError("404")], + ) as mock_gb, mock.patch( + "circup.command_utils.json" + ) as mock_json, mock.patch( + "circup.click.secho" + ): + mock_json.load.return_value = tags_data + bundle = circup.Bundle(TEST_BUNDLE_NAME) + bundle.platform = "10mpy" + tags = tags_data_load() + bundle.available_tags = tags.get(bundle.key, []) + ensure_latest_bundle(bundle) + mock_gb.assert_called_with(bundle, "54321", "10mpy") + assert bundle.current_tag == "54321" + assert bundle.platform is None + assert mock_json.dump.call_count == 1 + + def test_get_bundle(): """ Ensure the expected calls are made to get the referenced bundle and the @@ -1095,20 +1123,18 @@ def test_get_bundle(): "circup.command_utils.zipfile" ) as mock_zipfile, mock.patch( "circup.Bundle.add_tag" - ) as mock_add_tag: + ): mock_click.progressbar = mock_progress mock_requests.get().status_code = mock_requests.codes.ok mock_requests.get.reset_mock() tag = "12345" bundle = circup.Bundle(TEST_BUNDLE_NAME) - get_bundle(bundle, tag) - # how many bundles currently supported. i.e. 6x.mpy, 7x.mpy, py = 3 bundles - _bundle_count = len(PLATFORMS) + get_bundle(bundle, tag, "py") + _bundle_count = 1 assert mock_requests.get.call_count == _bundle_count assert mock_open.call_count == _bundle_count assert mock_zipfile.ZipFile.call_count == _bundle_count assert mock_zipfile.ZipFile().__enter__().extractall.call_count == _bundle_count - assert mock_add_tag.call_count == 1 def test_get_bundle_network_error(): @@ -1127,7 +1153,7 @@ def test_get_bundle_network_error(): tag = "12345" with pytest.raises(Exception) as ex: bundle = circup.Bundle(TEST_BUNDLE_NAME) - get_bundle(bundle, tag) + get_bundle(bundle, tag, "py") assert ex.value.args[0] == "Bang!" url = ( "https://github.com/" + TEST_BUNDLE_NAME + "/releases/download"