Merge pull request #624 from fetchai/develop

Release v0.1.16

HISTORY.rst

@@ -144,3 +144,21 @@ Release History
 - Multiple minor doc updates
 - Adds additional tests
 - Multiple additional minor fixes and changes
+
+0.1.16 (2020-01-12)
+-------------------
+
+- Completes tac skills implementation
+- Adds default ledger field to agent config
+- Converts ledger apis to dictionary fields in agent config
+- Introduces public ids to CLI and deprecate usage of package names only
+- Adds local push and public commands to CLI
+- Introduces ledger api abstract class
+- Unifies import paths for static and dynamic imports
+- Disambiguates import paths by introducing pattern of `packages.author.package_type_pluralized.package_name`
+- Adds agent directory to packages with some samples
+- Adds protocol generator and exposes on CLI
+- Removes unused config fields
+- Updates docs to align with recent changes
+- Adds additional tests on CLI
+- Multiple additional minor fixes and changes

aea/__version__.py

@@ -23,7 +23,7 @@
 __title__ = 'aea'
 __description__ = 'Autonomous Economic Agent framework'
 __url__ = 'https://github.com/fetchai/agents-aea.git'
-__version__ = '0.1.15'
+__version__ = '0.1.16'
 __author__ = 'Fetch.AI Limited'
 __license__ = 'Apache 2.0'
 __copyright__ = '2019 Fetch.AI Limited'

aea/cli/common.py

@@ -19,7 +19,6 @@
 
 """Implementation of the common utils of the aea cli."""
 
-import importlib.util
 import logging
 import logging.config
 import os
@@ -33,15 +32,19 @@
 
 from aea.cli.loggers import default_logging_config
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE, AgentConfig, SkillConfig, ConnectionConfig, ProtocolConfig, \
-    DEFAULT_PROTOCOL_CONFIG_FILE, DEFAULT_CONNECTION_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, Dependencies
+    DEFAULT_PROTOCOL_CONFIG_FILE, DEFAULT_CONNECTION_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, Dependencies, PublicId
 from aea.configurations.loader import ConfigLoader
+from aea.crypto.fetchai import FETCHAI
+from aea.helpers.base import add_agent_component_module_to_sys_modules, load_agent_component_package
 
 logger = logging.getLogger("aea")
 logger = default_logging_config(logger)
 
-DEFAULT_REGISTRY_PATH = str(Path("..", "packages"))
-DEFAULT_CONNECTION = "stub"
-DEFAULT_SKILL = "error"
+DEFAULT_VERSION = "0.1.0"
+DEFAULT_CONNECTION = PublicId.from_string("fetchai/stub:" + DEFAULT_VERSION)  # type: PublicId
+DEFAULT_SKILL = PublicId.from_string("fetchai/error:" + DEFAULT_VERSION)  # type: PublicId
+DEFAULT_LEDGER = FETCHAI
+DEFAULT_REGISTRY_PATH = str(Path("./", "packages"))
 
 
 class Context(object):
@@ -76,19 +79,19 @@ def get_dependencies(self) -> Dependencies:
         """
         dependencies = {}  # type: Dependencies
         for protocol_id in self.agent_config.protocols:
-            path = str(Path("protocols", protocol_id, DEFAULT_PROTOCOL_CONFIG_FILE))
+            path = str(Path("protocols", protocol_id.name, DEFAULT_PROTOCOL_CONFIG_FILE))
             protocol_config = self.protocol_loader.load(open(path))
             deps = cast(Dependencies, protocol_config.dependencies)
             dependencies.update(deps)
 
         for connection_id in self.agent_config.connections:
-            path = str(Path("connections", connection_id, DEFAULT_CONNECTION_CONFIG_FILE))
+            path = str(Path("connections", connection_id.name, DEFAULT_CONNECTION_CONFIG_FILE))
             connection_config = self.connection_loader.load(open(path))
             deps = cast(Dependencies, connection_config.dependencies)
             dependencies.update(deps)
 
         for skill_id in self.agent_config.skills:
-            path = str(Path("skills", skill_id, DEFAULT_SKILL_CONFIG_FILE))
+            path = str(Path("skills", skill_id.name, DEFAULT_SKILL_CONFIG_FILE))
             skill_config = self.skill_loader.load(open(path))
             deps = cast(Dependencies, skill_config.dependencies)
             dependencies.update(deps)
@@ -99,36 +102,46 @@ def get_dependencies(self) -> Dependencies:
 pass_ctx = click.make_pass_decorator(Context)
 
 
-def _try_to_load_agent_config(ctx: Context):
+def try_to_load_agent_config(ctx: Context, exit_on_except: bool = True) -> None:
+    """
+    Load agent config to a click context object.
+
+    :param ctx: click command context object.
+    :param exit_on_except: bool option to exit on exception (default = True).
+
+    :return None
+    """
     try:
-        path = Path(DEFAULT_AEA_CONFIG_FILE)
-        fp = open(str(path), mode="r", encoding="utf-8")
-        ctx.agent_config = ctx.agent_loader.load(fp)
-        logging.config.dictConfig(ctx.agent_config.logging_config)
+        path = Path(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE))
+        with open(str(path), mode="r", encoding="utf-8") as fp:
+            ctx.agent_config = ctx.agent_loader.load(fp)
+            logging.config.dictConfig(ctx.agent_config.logging_config)
     except FileNotFoundError:
-        logger.error("Agent configuration file '{}' not found in the current directory.".format(DEFAULT_AEA_CONFIG_FILE))
-        sys.exit(1)
+        if exit_on_except:
+            logger.error("Agent configuration file '{}' not found in the current directory.".format(DEFAULT_AEA_CONFIG_FILE))
+            sys.exit(1)
     except jsonschema.exceptions.ValidationError:
-        logger.error("Agent configuration file '{}' is invalid. Please check the documentation.".format(DEFAULT_AEA_CONFIG_FILE))
-        sys.exit(1)
+        if exit_on_except:
+            logger.error("Agent configuration file '{}' is invalid. Please check the documentation.".format(DEFAULT_AEA_CONFIG_FILE))
+            sys.exit(1)
 
 
 def _try_to_load_protocols(ctx: Context):
-    for protocol_name in ctx.agent_config.protocols:
-        logger.debug("Processing protocol {}".format(protocol_name))
+    for protocol_public_id in ctx.agent_config.protocols:
+        protocol_name = protocol_public_id.name
+        protocol_author = protocol_public_id.author
+        logger.debug("Processing protocol {}".format(protocol_public_id))
         try:
             ctx.protocol_loader.load(open(os.path.join("protocols", protocol_name, DEFAULT_PROTOCOL_CONFIG_FILE)))
         except FileNotFoundError:
             logger.error("Protocol configuration file for protocol {} not found.".format(protocol_name))
             sys.exit(1)
 
         try:
-            protocol_spec = importlib.util.spec_from_file_location(protocol_name, os.path.join("protocols", protocol_name, "__init__.py"))
-            protocol_module = importlib.util.module_from_spec(protocol_spec)
-            protocol_spec.loader.exec_module(protocol_module)  # type: ignore
-            sys.modules[protocol_spec.name + "_protocol"] = protocol_module
+            protocol_package = load_agent_component_package("protocol", protocol_name, protocol_author, Path("protocols", protocol_name))
+            add_agent_component_module_to_sys_modules("protocol", protocol_name, protocol_author, protocol_package)
         except Exception:
-            logger.error("A problem occurred while processing protocol {}.".format(protocol_name))
+            logger.error("A problem occurred while processing protocol {}.".format(protocol_public_id))
             sys.exit(1)
 
 
@@ -151,12 +164,13 @@ def format_items(items):
             'Public ID: {public_id}\n'
             'Name: {name}\n'
             'Description: {description}\n'
+            'Author: {author}\n'
             'Version: {version}\n'
             '{line}\n'.format(
                 name=item['name'],
-                # TODO: switch to unsafe get public_id when every obj has it
-                public_id=item.get('public_id'),
+                public_id=item['public_id'],
                 description=item['description'],
+                author=item['author'],
                 version=item['version'],
                 line='-' * 30
             ))
@@ -189,10 +203,12 @@ def format_skills(items):
 
 
 def retrieve_details(name: str, loader: ConfigLoader, config_filepath: str):
-    """Return description of a protocol, skill or connection."""
+    """Return description of a protocol, skill, connection."""
     config = loader.load(open(str(config_filepath)))
-    assert config.name == name
-    return {"name": config.name, "description": config.description, "version": config.version}
+    item_name = config.agent_name if isinstance(config, AgentConfig) else config.name
+    assert item_name == name
+    return {"public_id": str(config.public_id), "name": item_name, "author": config.author,
+            "description": config.description, "version": config.version}
 
 
 class AEAConfigException(Exception):
@@ -222,3 +238,62 @@ def arg_strip(s):
             return list(connection_names)
         except Exception:  # pragma: no cover
             raise click.BadParameter(value)
+
+
+class PublicIdParameter(click.ParamType):
+    """Define a public id parameter for Click applications."""
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the Public Id parameter.
+
+        Just forwards arguments to parent constructor.
+        """
+        super().__init__(*args, **kwargs)
+
+    def get_metavar(self, param):
+        """Return the metavar default for this param if it provides one."""
+        return "PUBLIC_ID"
+
+    def convert(self, value, param, ctx):
+        """Convert the value. This is not invoked for values that are `None` (the missing value)."""
+        try:
+            return PublicId.from_string(value)
+        except ValueError:
+            raise click.BadParameter(value)
+
+
+def try_get_item_source_path(path: str, item_type_plural: str, item_name: str) -> str:
+    """
+    Get the item source path.
+
+    :param path: the source path root
+    :param item_type_plural: the item type (plural)
+    :param item_name: the item name
+
+    :return: the item source path
+    """
+    source_path = os.path.join(path, item_type_plural, item_name)
+    if not os.path.exists(source_path):
+        raise click.ClickException(
+            'Item "{}" not found in source folder.'.format(item_name)
+        )
+    return source_path
+
+
+def try_get_item_target_path(path: str, item_type_plural: str, item_name: str) -> str:
+    """
+    Get the item target path.
+
+    :param path: the target path root
+    :param item_type_plural: the item type (plural)
+    :param item_name: the item name
+
+    :return: the item target path
+    """
+    target_path = os.path.join(path, item_type_plural, item_name)
+    if os.path.exists(target_path):
+        raise click.ClickException(
+            'Item "{}" already exists in target folder.'.format(item_name)
+        )
+    return target_path

aea/cli/config.py

@@ -25,10 +25,10 @@
 import click
 import yaml
 
-from aea.cli.common import Context, pass_ctx, _try_to_load_agent_config, logger
+from aea.cli.common import Context, pass_ctx, try_to_load_agent_config, logger
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, DEFAULT_PROTOCOL_CONFIG_FILE, \
-    DEFAULT_CONNECTION_CONFIG_FILE
-from aea.configurations.loader import ConfigLoader, ConfigurationType
+    DEFAULT_CONNECTION_CONFIG_FILE, ConfigurationType
+from aea.configurations.loader import ConfigLoader
 
 ALLOWED_PATH_ROOTS = ["agent", "skills", "protocols", "connections"]
 RESOURCE_TYPE_TO_CONFIG_FILE = {
@@ -113,7 +113,7 @@ def _get_parent_object(obj: dict, dotted_path: List[str]):
 @pass_ctx
 def config(ctx: Context):
     """Read or modify a configuration."""
-    _try_to_load_agent_config(ctx)
+    try_to_load_agent_config(ctx)
 
 
 @config.command()

aea/cli/core.py

@@ -29,10 +29,11 @@
 
 import aea
 from aea.cli.add import add
-from aea.cli.common import Context, pass_ctx, logger, _try_to_load_agent_config
+from aea.cli.common import Context, pass_ctx, logger, try_to_load_agent_config
 from aea.cli.config import config
 from aea.cli.create import create
 from aea.cli.fetch import fetch
+from aea.cli.generate import generate
 from aea.cli.install import install
 from aea.cli.list import list as _list
 from aea.cli.loggers import simple_verbosity_option
@@ -67,20 +68,20 @@ def delete(ctx: Context, agent_name):
     """Delete an agent."""
     path = Path(agent_name)
 
-    # check that the target folder is an AEA project.
     cwd = os.getcwd()
     try:
+        # check that the target folder is an AEA project.
         os.chdir(agent_name)
         fp = open(DEFAULT_AEA_CONFIG_FILE, mode="r", encoding="utf-8")
         ctx.agent_config = ctx.agent_loader.load(fp)
-        _try_to_load_agent_config(ctx)
+        try_to_load_agent_config(ctx)
     except Exception:
         logger.error("The name provided is not an AEA project.")
         sys.exit(1)
     finally:
         os.chdir(cwd)
 
-    logger.info("Deleting agent project directory '/{}'...".format(path))
+    logger.info("Deleting AEA project directory '/{}'...".format(path))
 
     # delete the agent's directory
     try:
@@ -94,7 +95,7 @@ def delete(ctx: Context, agent_name):
 @pass_ctx
 def freeze(ctx: Context):
     """Get the dependencies."""
-    _try_to_load_agent_config(ctx)
+    try_to_load_agent_config(ctx)
     for dependency_name, dependency_data in sorted(ctx.get_dependencies().items(), key=lambda x: x[0]):
         print(dependency_name + dependency_data.get("version", ""))
 
@@ -148,7 +149,7 @@ def _can_write(path) -> bool:
 @pass_ctx
 def add_key(ctx: Context, type_, file):
     """Add a private key to the wallet."""
-    _try_to_load_agent_config(ctx)
+    try_to_load_agent_config(ctx)
     _validate_private_key_path(file, type_)
     try:
         ctx.agent_config.private_key_paths.create(type_, PrivateKeyPathConfig(type_, file))
@@ -157,16 +158,17 @@ def add_key(ctx: Context, type_, file):
     ctx.agent_loader.dump(ctx.agent_config, open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w"))
 
 
-cli.add_command(create)
-cli.add_command(add)
 cli.add_command(_list)
-cli.add_command(login)
-cli.add_command(search)
+cli.add_command(add)
+cli.add_command(create)
 cli.add_command(config)
-cli.add_command(scaffold)
-cli.add_command(remove)
+cli.add_command(fetch)
+cli.add_command(generate)
 cli.add_command(install)
-cli.add_command(run)
-cli.add_command(push)
+cli.add_command(login)
 cli.add_command(publish)
-cli.add_command(fetch)
+cli.add_command(push)
+cli.add_command(remove)
+cli.add_command(run)
+cli.add_command(scaffold)
+cli.add_command(search)

aea/cli/create.py

@@ -30,7 +30,7 @@
 
 import aea
 from aea.cli.add import connection, skill
-from aea.cli.common import Context, logger, DEFAULT_REGISTRY_PATH, DEFAULT_CONNECTION, DEFAULT_SKILL
+from aea.cli.common import Context, logger, DEFAULT_REGISTRY_PATH, DEFAULT_CONNECTION, DEFAULT_SKILL, DEFAULT_LEDGER, DEFAULT_VERSION
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE, AgentConfig
 
 
@@ -75,19 +75,21 @@ def create(click_context, agent_name):
         # create a config file inside it
         logger.info("Creating config file {}".format(DEFAULT_AEA_CONFIG_FILE))
         config_file = open(os.path.join(agent_name, DEFAULT_AEA_CONFIG_FILE), "w")
-        agent_config = AgentConfig(agent_name=agent_name, aea_version=aea.__version__, author="", version="v1", license="", fingerprint="", url="", registry_path=DEFAULT_REGISTRY_PATH, description="")
+        agent_config = AgentConfig(agent_name=agent_name, aea_version=aea.__version__,
+                                   author="", version=DEFAULT_VERSION, license="", fingerprint="",
+                                   registry_path=os.path.join("..", DEFAULT_REGISTRY_PATH), description="")
         agent_config.default_connection = DEFAULT_CONNECTION
+        agent_config.default_ledger = DEFAULT_LEDGER
         ctx.agent_loader.dump(agent_config, config_file)
 
         # next commands must be done from the agent's directory -> overwrite ctx.cwd
         ctx.agent_config = agent_config
         ctx.cwd = agent_config.agent_name
 
-        logger.info("Default connections:")
-        click_context.invoke(connection, connection_name=DEFAULT_CONNECTION)
+        logger.info("Adding default packages ...")
+        click_context.invoke(connection, connection_public_id=DEFAULT_CONNECTION)
 
-        logger.info("Default skills:")
-        click_context.invoke(skill, skill_name=DEFAULT_SKILL)
+        click_context.invoke(skill, skill_public_id=DEFAULT_SKILL)
 
     except OSError:
         logger.error("Directory already exist. Aborting...")