jupe · jupe · Aug 16, 2021 · Aug 7, 2021 · Aug 7, 2021 · Aug 7, 2021
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,6 @@
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    @abstractmethod
diff --git a/README.md b/README.md
@@ -19,6 +19,11 @@ Resource is released in following cases:
 * allocation.unlock() is called
 * lockable.unlock(<allocation>) is called
 
+Resources data provider support following mechanisms:
+* `resources.json` file in file system
+* python list of dictionaries
+* http uri which points to API and is used with HTTP GET method. API should provide `resources.json` data as json object.
+
 # CLI interface
 
 ```
@@ -52,7 +57,7 @@ optional arguments:
 
 Constructor
 ```python
-lockable = Lockable([hostname], [resource_list_file], [lock_folder])
+lockable = Lockable([hostname], [resource_list_file], [resource_list], [lock_folder])
 ```
 
 Allocation

diff --git a/lockable/__init__.py b/lockable/__init__.py
@@ -1,2 +1,3 @@
 """ Lockable module """
-from lockable.lockable import Lockable, ResourceNotFound, Allocation
+from lockable.lockable import Lockable, ResourceNotFound, Allocation, MODULE_LOGGER
+from lockable.provider import Provider, ProviderError
diff --git a/lockable/cli.py b/lockable/cli.py
@@ -21,15 +21,15 @@ def get_args():
         formatter_class=argparse.RawTextHelpFormatter)
 
     parser.add_argument('--validate-only',
-                        action = "store_true",
+                        action="store_true",
                         default=False,
                         help='Only validate resources.json')
     parser.add_argument('--lock-folder',
                         default='.',
                         help='lock folder')
     parser.add_argument('--resources',
                         default='./resources.json',
-                        help='Resources file')
+                        help='Resources file or http uri')
     parser.add_argument('--timeout',
                         default=1,
                         help='Timeout for trying allocate suitable resource')

diff --git a/lockable/lockable.py b/lockable/lockable.py
@@ -10,8 +10,9 @@
 from dataclasses import dataclass
 from contextlib import contextmanager
 from uuid import uuid1
-from pydash import filter_, merge, count_by
+from pydash import filter_, merge
 from pid import PidFile, PidFileError
+from lockable.provider_helpers import create as create_provider
 
 MODULE_LOGGER = logging.getLogger('lockable')
 
@@ -63,68 +64,18 @@ def __init__(self, hostname=socket.gethostname(),
         self.logger.debug('Initialized lockable')
         self._hostname = hostname
         self._lock_folder = lock_folder
-        self._resource_list = None
-        self._resource_list_file_mtime = None
-        self._resource_list_file = resource_list_file
         assert not (isinstance(resource_list, list) and
                     resource_list_file), 'only one of resource_list or ' \
                                          'resource_list_file is accepted, not both'
-        if isinstance(self._resource_list_file, str):
-            self._resource_list_file_mtime = os.path.getmtime(resource_list_file)
-            self.load_resources_list_file(self._resource_list_file)
-        elif isinstance(resource_list, list):
-            self.load_resources_list(resource_list)
+        if resource_list is None and resource_list_file is None:
+            self._provider = create_provider([])
         else:
-            self.logger.warning('resource_list_file or resource_list is not configured')
+            self._provider = create_provider(resource_list_file or resource_list)
 
-    def load_resources_list_file(self, filename: str):
-        """ Load resources list file"""
-        self.load_resources_list(self._read_resources_list(filename))
-        self.logger.warning('Use resources from %s file', filename)
-
-    def load_resources_list(self, resources_list: list):
-        """ Load resources list """
-        assert isinstance(resources_list, list), 'resources_list is not an list'
-        self._resource_list = resources_list
-        self.logger.debug('Resources loaded: ')
-        for resource in self._resource_list:
-            self.logger.debug(json.dumps(resource))
-
-    def reload_resource_list_file(self):
-        """ Reload resources from file if file has been modified """
-        if self._resource_list_file_mtime is None:
-            return
-
-        mtime = os.path.getmtime(self._resource_list_file)
-        if self._resource_list_file_mtime != mtime:
-            self._resource_list_file_mtime = mtime
-            self.load_resources_list_file(self._resource_list_file)
-
-    @staticmethod
-    def _read_resources_list(filename):
-        """ Read resources json file """
-        MODULE_LOGGER.debug('Read resource list file: %s', filename)
-        with open(filename) as json_file:
-            try:
-                data = json.load(json_file)
-                assert isinstance(data, list), 'data is not an list'
-            except (json.decoder.JSONDecodeError, AssertionError) as error:
-                raise ValueError(f'invalid resources json file: {error}') from error
-            Lockable._validate_json(data)
-        return data
-
-    @staticmethod
-    def _validate_json(data):
-        """ Internal method to validate resources.json content """
-        counts = count_by(data, lambda obj: obj.get('id'))
-        no_ids = filter_(counts.keys(), lambda key: key is None)
-        if no_ids:
-            raise ValueError('Invalid json, id property is missing')
+    @property
+    def _resource_list(self):
+        return self._provider.data
 
-        duplicates = filter_(counts.keys(), lambda key: counts[key] > 1)
-        if duplicates:
-            MODULE_LOGGER.warning('Duplicates: %s', duplicates)
-            raise ValueError(f"Invalid json, duplicate ids in {duplicates}")
 
     @staticmethod
     def parse_requirements(requirements_str: (str or dict)) -> dict:
@@ -228,9 +179,10 @@ def lock(self, requirements: (str or dict), timeout_s: int = 1000) -> Allocation
         :return: Allocation context
         """
         assert isinstance(self._resource_list, list), 'resources list is not loaded'
-        self.reload_resource_list_file()
         requirements = self.parse_requirements(requirements)
         predicate = self._get_requirements(requirements, self._hostname)
+        # Refresh resources data
+        self._provider.reload()
         self.logger.debug("Use lock folder: %s", self._lock_folder)
         self.logger.debug("Requirements: %s", json.dumps(predicate))
         self.logger.debug("Resource list: %s", json.dumps(self._resource_list))

diff --git a/lockable/provider.py b/lockable/provider.py
@@ -0,0 +1,64 @@
+""" Provider library """
+from abc import ABC, abstractmethod
+import json
+import typing
+from typing import List
+import logging
+
+from urllib.parse import urlparse
+from pydash import filter_, count_by
+
+MODULE_LOGGER = logging.getLogger('lockable')
+
+
+class ProviderError(Exception):
+    """ Provider error """
+
+
+class Provider(ABC):
+    """ Abstract Provider """
+    def __init__(self, uri: typing.Union[str, list]):
+        """ Provider constructor """
+        self._uri = uri
+        self._resources = list()
+        self.reload()
+
+    @property
+    def data(self) -> list:
+        """ Get resources list """
+        return self._resources
+
+    @staticmethod
+    def is_http_url(uri: str) -> bool:
+        """ Check if argument is url format"""
+        try:
+            result = urlparse(uri)
+            return all([result.scheme, result.netloc])
+        except:  # pylint: disable=bare-except
+            return False
+
+    @abstractmethod
+    def reload(self) -> None:  # pragma: no cover
+        """ Reload resources data"""
+
+    def set_resources_list(self, resources_list: list):
+        """ Load resources list """
+        assert isinstance(resources_list, list), 'resources_list is not an list'
+        Provider._validate_json(resources_list)
+        self._resources = resources_list
+        MODULE_LOGGER.debug('Resources loaded: ')
+        for resource in self._resources:
+            MODULE_LOGGER.debug(json.dumps(resource))
+
+    @staticmethod
+    def _validate_json(data: List[dict]):
+        """ Internal method to validate resources.json content """
+        counts = count_by(data, lambda obj: obj.get('id'))
+        no_ids = filter_(counts.keys(), lambda key: key is None)
+        if no_ids:
+            raise ValueError('Invalid json, id property is missing')
+
+        duplicates = filter_(counts.keys(), lambda key: counts[key] > 1)
+        if duplicates:
+            MODULE_LOGGER.warning('Duplicates: %s', duplicates)
+            raise ValueError(f"Invalid json, duplicate ids in {duplicates}")
diff --git a/lockable/provider_file.py b/lockable/provider_file.py
@@ -0,0 +1,43 @@
+""" resources Provider for file """
+import json
+import os
+from typing import List
+
+from lockable.provider import Provider, MODULE_LOGGER
+
+
+class ProviderFile(Provider):
+    """ ProviderFile interface """
+
+    def __init__(self, uri: str):
+        """
+        ProviderFile constructor
+        :param uri: file path
+        """
+        self._resource_list_file_mtime = None
+        super().__init__(uri)
+
+    def reload(self):
+        """ Load resources list file"""
+        self.reload_resource_list_file()
+        MODULE_LOGGER.warning('Use resources from %s file', self._uri)
+
+    def reload_resource_list_file(self):
+        """ Reload resources from file if file has been modified """
+        mtime = os.path.getmtime(self._uri)
+        if self._resource_list_file_mtime != mtime:
+            self._resource_list_file_mtime = mtime
+            data = self._read_resources_list_file(self._uri)
+            self.set_resources_list(data)
+
+    @staticmethod
+    def _read_resources_list_file(filename: str) -> List[dict]:
+        """ Read resources json file """
+        MODULE_LOGGER.debug('Read resource list file: %s', filename)
+        with open(filename) as json_file:
+            try:
+                data = json.load(json_file)
+                assert isinstance(data, list), 'data is not an list'
+            except (json.decoder.JSONDecodeError, AssertionError) as error:
+                raise ValueError(f'invalid resources json file: {error}') from error
+        return data
diff --git a/lockable/provider_helpers.py b/lockable/provider_helpers.py
@@ -0,0 +1,21 @@
+""" resources Provider helper """
+from lockable.provider import Provider
+from lockable.provider_list import ProviderList
+from lockable.provider_file import ProviderFile
+from lockable.provider_http import ProviderHttp
+
+
+def create(uri):
+    """
+    Create provider instance from uri
+    :param uri: list of string for provider
+    :return: Provider object
+    :rtype: Provider
+    """
+    if Provider.is_http_url(uri):
+        return ProviderHttp(uri)
+    if isinstance(uri, str):
+        return ProviderFile(uri)
+    if isinstance(uri, list):
+        return ProviderList(uri)
+    raise AssertionError('uri should be list or string')
diff --git a/lockable/provider_http.py b/lockable/provider_http.py
@@ -0,0 +1,40 @@
+""" resources Provider for HTTP """
+import logging
+import requests
+from requests import HTTPError
+
+from lockable.provider import Provider, ProviderError
+
+MODULE_LOGGER = logging.getLogger('lockable')
+
+
+class ProviderHttp(Provider):
+    """ ProviderHttp interface"""
+
+    def __init__(self, uri: str):
+        """ ProviderHttp constructor """
+        super().__init__(uri)
+
+    def reload(self) -> None:
+        """ Reload resources list from web server """
+        self.set_resources_list(self._get_http(self._uri))
+
+    @staticmethod
+    def _get_http(uri: str) -> list:
+        """ Internal method to get http json data"""
+        try:
+            response = requests.get(uri)
+            response.raise_for_status()
+
+            # could utilise ETag or Last-Modified headers to optimize performance
+            # etag = response.headers.get("ETag")
+            # last_modified = response.headers.get("Last-Modified")
+
+            # access JSON content
+            return response.json()
+        except HTTPError as http_err:
+            MODULE_LOGGER.error('HTTP error occurred %s', http_err)
+            raise ProviderError(http_err.response.reason) from http_err
+        except Exception as err:
+            MODULE_LOGGER.error('Other error occurred: %s', err)
+            raise ProviderError(err) from err
diff --git a/lockable/provider_list.py b/lockable/provider_list.py
@@ -0,0 +1,14 @@
+""" resources Provider for static list """
+from lockable.provider import Provider
+
+
+class ProviderList(Provider):
+    """ ProviderList implementation """
+
+    def __init__(self, uri: list):
+        """ ProviderList constructor """
+        super().__init__(uri)
+        self.set_resources_list(self._uri)
+
+    def reload(self):
+        """ Nothing to do """
diff --git a/lockable/resources.json b/lockable/resources.json
@@ -0,0 +1,20 @@
+[
+  {
+    "id": "2",
+    "online": false,
+    "hostname": "localhost",
+    "test": true
+  },
+  {
+    "id": "1",
+    "online": true,
+    "hostname": "localhost",
+    "test": false
+  },
+  {
+    "id": "3",
+    "online": true,
+    "hostname": "localhost2",
+    "test": true
+  }
+]
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,11 @@
+certifi==2021.5.30
+charset-normalizer==2.0.4
+coverage==5.5
+httptest==0.0.17
+idna==3.2
+mock==4.0.3
+pid==3.0.4
+pydash==5.0.0
+requests==2.26.0
+setuptools-scm==6.0.1
+urllib3==1.26.6
diff --git a/setup.py b/setup.py
@@ -56,7 +56,9 @@
     },
     install_requires=[
         'pid',
-        'pydash'
+        'pydash',
+        'requests',
+        'httptest'
     ],
     extras_require={
         'dev': ['nose', 'coveralls', 'pylint', 'coverage', 'mock'],