reckless: remove remaining traces of API access

Including the canned server in the pytest infrastructure.

Author: endothermicdev

tests/data/recklessrepo/rkls_api_lightningd_plugins.json

@@ -1,15 +1,13 @@
-from fixtures import *  # noqa: F401,F403
-import subprocess
-from pathlib import PosixPath, Path
-import socket
-from pyln.testing.utils import VALGRIND
 import json
-import pytest
 import os
+from pathlib import PosixPath, Path
 import re
-import shutil
+import subprocess
 import time
 import unittest
+from fixtures import *  # noqa: F401,F403
+from pyln.testing.utils import VALGRIND
+import pytest
 
 
 @pytest.fixture(autouse=True)
@@ -22,20 +20,10 @@ def canned_github_server(directory):
     if os.environ.get('LIGHTNING_CLI') is None:
         os.environ['LIGHTNING_CLI'] = str(FILE_PATH.parent / 'cli/lightning-cli')
         print('LIGHTNING_CALL: ', os.environ.get('LIGHTNING_CLI'))
-    # Use socket to provision a random free port
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.bind(('localhost', 0))
-    free_port = str(sock.getsockname()[1])
-    sock.close()
     global my_env
     my_env = os.environ.copy()
-    # This tells reckless to redirect to the canned server rather than github.
-    my_env['REDIR_GITHUB_API'] = f'http://127.0.0.1:{free_port}/api'
+    # This tells reckless to redirect to the local test plugins repo rather than github.
     my_env['REDIR_GITHUB'] = directory
-    my_env['FLASK_RUN_PORT'] = free_port
-    my_env['FLASK_APP'] = str(FILE_PATH / 'rkls_github_canned_server')
-    server = subprocess.Popen(["python3", "-m", "flask", "run"],
-                              env=my_env)
 
     # Generate test plugin repository to test reckless against.
     repo_dir = os.path.join(directory, "lightningd")
@@ -84,13 +72,10 @@ def canned_github_server(directory):
     del my_env['GIT_DIR']
     del my_env['GIT_WORK_TREE']
     del my_env['GIT_INDEX_FILE']
-    # We also need the github api data for the repo which will be served via http
-    shutil.copyfile(str(FILE_PATH / 'data/recklessrepo/rkls_api_lightningd_plugins.json'), os.path.join(directory, 'rkls_api_lightningd_plugins.json'))
     yield
     # Delete requirements.txt from the testplugpass directory
     with open(requirements_file_path, 'w') as f:
         f.write(f"pyln-client\n\n")
-    server.terminate()
 
 
 class RecklessResult:

tests/test_reckless.py

@@ -18,7 +18,6 @@ import types
 from typing import Union
 from urllib.parse import urlparse
 from urllib.request import urlopen
-from urllib.error import HTTPError
 import venv
 
 
@@ -258,7 +257,7 @@ class Source(Enum):
     REMOTE_GIT_REPO = 3
     OTHER_URL = 4
     UNKNOWN = 5
-    # Cloned from remote source before searching (rather than github API)
+    # Cloned from remote source before searching
     GIT_LOCAL_CLONE = 6
 
     @classmethod
@@ -302,7 +301,7 @@ class SubmoduleSource:
 class LoadedSource:
     """Allows loading all sources only once per call of reckless.  Initialized
     with a single line of the reckless .sources file.  Keeping state also allows
-    minimizing API calls and refetching repositories."""
+    minimizing refetching repositories."""
     def __init__(self, source: str):
         self.original_source = source
         self.type = Source.get_type(source)
@@ -445,16 +444,12 @@ class InstInfo:
         # Set recursion for how many directories deep we should search
         depth = 0
         if self.srctype in [Source.DIRECTORY, Source.LOCAL_REPO,
-                            Source.GIT_LOCAL_CLONE]:
+                            Source.GIT_LOCAL_CLONE, Source.REMOTE_GIT_REPO]:
             depth = 5
-        elif self.srctype == Source.REMOTE_GIT_REPO:
-            depth = 1
 
         def search_dir(self, sub: SourceDir, subdir: bool,
                        recursion: int) -> Union[SourceDir, None]:
             assert isinstance(recursion, int)
-            # carveout for archived plugins in lightningd/plugins. Other repos
-            # are only searched by API at the top level.
             if recursion == 0 and 'archive' in sub.name.lower():
                 pass
             # If unable to search deeper, resort to matching directory name
@@ -501,27 +496,7 @@ class InstInfo:
                         return success
             return None
 
-        try:
-            result = search_dir(self, target, False, depth)
-        # Using the rest API of github.com may result in a
-        # "Error 403: rate limit exceeded" or other access issues.
-        # Fall back to cloning and searching the local copy instead.
-        except HTTPError:
-            result = None
-            if self.srctype == Source.REMOTE_GIT_REPO:
-                # clone source to reckless dir
-                target = copy_remote_git_source(self)
-                if not target:
-                    log.warning(f"could not clone github source {self}")
-                    return False
-                log.debug(f"falling back to cloning remote repo {self}")
-                # Update to reflect use of a local clone
-                self.source_loc = str(target.location)
-                self.srctype = target.srctype
-                result = search_dir(self, target, False, 5)
-
-                if not result:
-                    return False
+        result = search_dir(self, target, False, depth)
 
         if result:
             if result != target:
@@ -1217,8 +1192,7 @@ def help_alias(targets: list):
 
 
 def _get_local_clone(source: str) -> Union[Path, None]:
-    """Returns the path of a local repository clone of a github source. If one
-    already exists, prefer searching that to accessing the github API."""
+    """Returns the path of a local repository clone of a github source."""
     user, repo = Source.get_github_user_repo(source)
     local_clone_location = RECKLESS_DIR / '.remote_sources' / user / repo
     if local_clone_location.exists():
@@ -1232,8 +1206,7 @@ def _source_search(name: str, src: LoadedSource) -> Union[InstInfo, None]:
     root_dir = src.content
     source = InstInfo(name, root_dir.location)
 
-    # If a local clone of a github source already exists, prefer searching
-    # that instead of accessing the github API.
+    # Remote git sources require a local clone before searching.
     if src.type == Source.REMOTE_GIT_REPO:
         if src.local_clone:
             if not src.local_clone_fetched:
@@ -2454,18 +2427,11 @@ if __name__ == '__main__':
     LIGHTNING_CONFIG = args.conf
     RECKLESS_CONFIG = load_config(reckless_dir=str(RECKLESS_DIR),
                                   network=NETWORK)
-    API_GITHUB_COM = 'https://api.github.com'
     GITHUB_COM = 'https://github.com'
     # Used for blackbox testing to avoid hitting github servers
-    if 'REDIR_GITHUB_API' in os.environ:
-        API_GITHUB_COM = os.environ['REDIR_GITHUB_API']
     if 'REDIR_GITHUB' in os.environ:
         GITHUB_COM = os.environ['REDIR_GITHUB']
 
-    GITHUB_API_FALLBACK = False
-    if 'GITHUB_API_FALLBACK' in os.environ:
-        GITHUB_API_FALLBACK = os.environ['GITHUB_API_FALLBACK']
-
     RECKLESS_SOURCES = load_sources()
     if 'targets' in args: # and len(args.targets) > 0:
         if args.func.__name__ == 'help_alias':