Add support for GET /v2/dataframes API endpoint to the client

Support for the ``GET /v2/dataframes`` endpoint has been added
to the client. A new ``dataframes get`` CLI command is also available.

Story: 2005890
Task: 36384
Depends-On: https://review.opendev.org/#/c/679636
Change-Id: Idfe93025e0f740906d0f53f33547c7746fc15169

.zuul.yaml

@@ -1,5 +1,5 @@
 - job:
-    name: cloudkittyclient-devstack-functional
+    name: cloudkittyclient-devstack-functional-base
     parent: devstack
     description: |
       Job for cloudkittyclient functional tests
@@ -21,40 +21,43 @@
         cloudkitty: https://opendev.org/openstack/cloudkitty
       devstack_localrc:
         CLOUDKITTY_FETCHER: keystone
+        DEVSTACK_GATE_USE_PYTHON3: "True"
+        USE_PYTHON3: True
       devstack_services:
         ck-api: true
         horizon: false
       tox_install_siblings: false
       zuul_work_dir: src/opendev.org/openstack/python-cloudkittyclient
-      tox_envlist: functional
 
 - job:
-    name: cloudkittyclient-devstack-functional-py3
-    parent: cloudkittyclient-devstack-functional
-    description: |
-      Job for cloudkittyclient functional tests, ran in python3.
+    name: cloudkittyclient-devstack-functional-v1-client
+    parent: cloudkittyclient-devstack-functional-base
     vars:
-      devstack_localrc:
-        DEVSTACK_GATE_USE_PYTHON3: "True"
-        USE_PYTHON3: "True"
+      tox_envlist: functional-v1
+
+- job:
+    name: cloudkittyclient-devstack-functional-v2-client
+    parent: cloudkittyclient-devstack-functional-base
+    vars:
+      tox_envlist: functional-v2
 
 - project:
     templates:
+      - openstack-lower-constraints-jobs
       - check-requirements
       - openstack-cover-jobs
-      - openstack-python-jobs
-      - openstack-python3-train-jobs
+      - openstack-python3-victoria-jobs
       - openstackclient-plugin-jobs
       - publish-openstack-docs-pti
     check:
       jobs:
-        - cloudkittyclient-devstack-functional:
+        - cloudkittyclient-devstack-functional-v1-client:
             voting: true
-        - cloudkittyclient-devstack-functional-py3:
+        - cloudkittyclient-devstack-functional-v2-client:
             voting: true
     gate:
       jobs:
-        - cloudkittyclient-devstack-functional:
+        - cloudkittyclient-devstack-functional-v1-client:
             voting: true
-        - cloudkittyclient-devstack-functional-py3:
+        - cloudkittyclient-devstack-functional-v2-client:
             voting: true

babel.cfg

@@ -1,2 +0,0 @@
-[python: **.py]
-

cloudkittyclient/common/base.py

@@ -12,13 +12,24 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-#
+
 from string import Formatter as StringFormatter
+from urllib.parse import urlencode
 
-from six.moves.urllib.parse import urlencode
+from cloudkittyclient import utils
 
 
-class BaseManager(object):
+class HttpDecoratorMeta(type):
+
+    ignore = ('get_url', )
+
+    def __new__(cls, *args, **kwargs):
+        return utils.format_http_errors(HttpDecoratorMeta.ignore)(
+            super(HttpDecoratorMeta, cls).__new__(cls, *args, **kwargs)
+        )
+
+
+class BaseManager(object, metaclass=HttpDecoratorMeta):
     """Base class for Endpoint Manager objects."""
 
     url = ''

cloudkittyclient/common/client.py

@@ -26,6 +26,9 @@ class BaseClient(object):
                  insecure=False,
                  **kwargs):
         adapter_options.setdefault('service_type', 'rating')
+        adapter_options.setdefault('additional_headers', {
+            'Content-Type': 'application/json',
+        })
 
         if insecure:
             verify_cert = False

cloudkittyclient/shell.py

@@ -94,7 +94,7 @@ class CloudKittyShell(cliff.app.App):
         super(CloudKittyShell, self).__init__(
             description='CloudKitty CLI client',
             version=utils.get_version(),
-            command_manager=CommandManager('cloudkittyclient.v{}'.format(
+            command_manager=CommandManager('cloudkittyclient_v{}'.format(
                 self._get_api_version(args[:]),
             )),
             deferred_help=True,

cloudkittyclient/tests/functional/base.py

@@ -24,24 +24,30 @@ from cloudkittyclient.tests import utils
 class BaseFunctionalTest(utils.BaseTestCase):
 
     def _run(self, executable, action,
-             flags='', params='', fmt='-f json', has_output=True):
+             flags='', params='', fmt='-f json', stdin=None, has_output=True):
         if not has_output:
             fmt = ''
         cmd = ' '.join([executable, flags, action, params, fmt])
         cmd = shlex.split(cmd)
-        p = subprocess.Popen(cmd, env=os.environ.copy(), shell=False,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
+        p = subprocess.Popen(
+            cmd, env=os.environ.copy(), shell=False,
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE if stdin else None,
+        )
+        stdout, stderr = p.communicate(input=stdin)
         if p.returncode != 0:
             raise RuntimeError('"{cmd}" returned {val}: {msg}'.format(
                 cmd=' '.join(cmd), val=p.returncode, msg=stderr))
         return json.loads(stdout) if has_output else None
 
     def openstack(self, action,
-                  flags='', params='', fmt='-f json', has_output=True):
+                  flags='', params='', fmt='-f json',
+                  stdin=None, has_output=True):
         return self._run('openstack rating', action,
-                         flags, params, fmt, has_output)
+                         flags, params, fmt, stdin, has_output)
 
     def cloudkitty(self, action,
-                   flags='', params='', fmt='-f json', has_output=True):
-        return self._run('cloudkitty', action, flags, params, fmt, has_output)
+                   flags='', params='', fmt='-f json',
+                   stdin=None, has_output=True):
+        return self._run('cloudkitty', action, flags, params, fmt,
+                         stdin, has_output)

cloudkittyclient/tests/functional/v2/test_dataframes.py

@@ -0,0 +1,174 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import os
+import uuid
+
+from cloudkittyclient.tests.functional import base
+
+
+class CkDataframesTest(base.BaseFunctionalTest):
+    dataframes_data = """
+    {
+        "dataframes": [
+            {
+                "period": {
+                    "begin": "20190723T122810Z",
+                    "end": "20190723T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 1.2
+                            },
+                            "rating": {
+                                "price": 0.04
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 200.4
+                            },
+                            "rating": {
+                                "price": 0.06
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            },
+            {
+                "period": {
+                    "begin": "20190823T122810Z",
+                    "end": "20190823T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 2.4
+                            },
+                            "rating": {
+                                "price": 0.08
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 400.8
+                            },
+                            "rating": {
+                                "price": 0.12
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(CkDataframesTest, self).__init__(*args, **kwargs)
+        self.runner = self.cloudkitty
+
+    def setUp(self):
+        super(CkDataframesTest, self).setUp()
+
+        self.fixture_file_name = '{}.json'.format(uuid.uuid4())
+        with open(self.fixture_file_name, 'w') as f:
+            f.write(self.dataframes_data)
+
+    def tearDown(self):
+        files = os.listdir('.')
+        if self.fixture_file_name in files:
+            os.remove(self.fixture_file_name)
+
+        super(CkDataframesTest, self).tearDown()
+
+    def test_dataframes_add_with_no_args(self):
+        self.assertRaisesRegexp(
+            RuntimeError,
+            'error: the following arguments are required: datafile',
+            self.runner,
+            'dataframes add',
+            fmt='',
+            has_output=False,
+        )
+
+    def test_dataframes_add(self):
+        self.runner(
+            'dataframes add {}'.format(self.fixture_file_name),
+            fmt='',
+            has_output=False,
+        )
+
+    def test_dataframes_add_with_hyphen_stdin(self):
+        with open(self.fixture_file_name, 'r') as f:
+            self.runner(
+                'dataframes add -',
+                fmt='',
+                stdin=f.read().encode(),
+                has_output=False,
+            )
+
+    def test_dataframes_get(self):
+        # TODO(jferrieu): functional tests will be added in another
+        # patch for `dataframes get`
+        pass
+
+
+class OSCDataframesTest(CkDataframesTest):
+    def __init__(self, *args, **kwargs):
+        super(OSCDataframesTest, self).__init__(*args, **kwargs)
+        self.runner = self.openstack

cloudkittyclient/tests/unit/v1/test_hashmap.py

@@ -13,7 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-import mock
+from unittest import mock
 
 from cloudkittyclient import exc
 from cloudkittyclient.tests.unit.v1 import base

cloudkittyclient/tests/unit/v1/test_report_cli.py

@@ -13,7 +13,7 @@
 #    under the License.
 #
 import collections
-import mock
+from unittest import mock
 
 from cloudkittyclient.tests.unit.v1 import base
 from cloudkittyclient.v1 import report_cli

cloudkittyclient/tests/unit/v2/base.py

@@ -12,7 +12,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+
 from cloudkittyclient.tests import utils
+from cloudkittyclient.v2 import dataframes
+from cloudkittyclient.v2.rating import modules
 from cloudkittyclient.v2 import scope
 from cloudkittyclient.v2 import summary
 
@@ -22,5 +25,7 @@ class BaseAPIEndpointTestCase(utils.BaseTestCase):
     def setUp(self):
         super(BaseAPIEndpointTestCase, self).setUp()
         self.api_client = utils.FakeHTTPClient()
+        self.dataframes = dataframes.DataframesManager(self.api_client)
         self.scope = scope.ScopeManager(self.api_client)
         self.summary = summary.SummaryManager(self.api_client)
+        self.rating = modules.RatingManager(self.api_client)

cloudkittyclient/tests/unit/v2/test_dataframes.py

@@ -0,0 +1,172 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import json
+
+from collections import OrderedDict
+
+from cloudkittyclient import exc
+from cloudkittyclient.tests.unit.v2 import base
+
+
+class TestDataframes(base.BaseAPIEndpointTestCase):
+    dataframes_data = """
+    {
+        "dataframes": [
+            {
+                "period": {
+                    "begin": "20190723T122810Z",
+                    "end": "20190723T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 1.2
+                            },
+                            "rating": {
+                                "price": 0.04
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 200.4
+                            },
+                            "rating": {
+                                "price": 0.06
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            },
+            {
+                "period": {
+                    "begin": "20190823T122810Z",
+                    "end": "20190823T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 2.4
+                            },
+                            "rating": {
+                                "price": 0.08
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 400.8
+                            },
+                            "rating": {
+                                "price": 0.12
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+    """
+
+    def test_add_dataframes_with_string(self):
+        self.dataframes.add_dataframes(
+            dataframes=self.dataframes_data,
+        )
+        self.api_client.post.assert_called_once_with(
+            '/v2/dataframes',
+            data=self.dataframes_data,
+        )
+
+    def test_add_dataframes_with_json_object(self):
+        json_data = json.loads(self.dataframes_data)
+
+        self.dataframes.add_dataframes(
+            dataframes=json_data,
+        )
+        self.api_client.post.assert_called_once_with(
+            '/v2/dataframes',
+            data=json.dumps(json_data),
+        )
+
+    def test_add_dataframes_with_neither_string_nor_object_raises_exc(self):
+        self.assertRaises(
+            exc.InvalidArgumentError,
+            self.dataframes.add_dataframes,
+            dataframes=[open],
+        )
+
+    def test_add_dataframes_with_no_args_raises_exc(self):
+        self.assertRaises(
+            exc.ArgumentRequired,
+            self.dataframes.add_dataframes)
+
+    def test_get_dataframes(self):
+        self.dataframes.get_dataframes()
+        self.api_client.get.assert_called_once_with('/v2/dataframes')
+
+    def test_get_dataframes_with_pagination_args(self):
+        self.dataframes.get_dataframes(offset=10, limit=10)
+        try:
+            self.api_client.get.assert_called_once_with(
+                '/v2/dataframes?limit=10&offset=10')
+        except AssertionError:
+            self.api_client.get.assert_called_once_with(
+                '/v2/dataframes?offset=10&limit=10')
+
+    def test_get_dataframes_filters(self):
+        self.dataframes.get_dataframes(
+            filters=OrderedDict([('one', 'two'), ('three', 'four')]))
+        self.api_client.get.assert_called_once_with(
+            '/v2/dataframes?filters=one%3Atwo%2Cthree%3Afour')

cloudkittyclient/tests/unit/v2/test_rating.py

@@ -0,0 +1,38 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+from cloudkittyclient.tests.unit.v2 import base
+
+
+class TestRating(base.BaseAPIEndpointTestCase):
+
+    def test_get_modules(self):
+        self.rating.get_module()
+        self.api_client.get.assert_called_once_with('/v2/rating/modules')
+
+    def test_get_one_module(self):
+        self.rating.get_module(module_id="moduleidtest")
+        self.api_client.get.assert_called_once_with(
+            '/v2/rating/modules/moduleidtest')
+
+    def test_update_one_module(self):
+        self.rating.update_module(module_id="moduleidtest",
+                                  enabled=False, priority=42)
+        self.api_client.put.assert_called_once_with(
+            '/v2/rating/modules/moduleidtest',
+            json={
+                'enabled': False,
+                'priority': 42,
+            })

cloudkittyclient/tests/utils.py

@@ -12,12 +12,13 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+from unittest import mock
+
 import fixtures
 import testtools
 
 from keystoneauth1 import adapter
 from keystoneauth1 import session
-import mock
 
 
 class BaseTestCase(testtools.TestCase):

cloudkittyclient/utils.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2018 Objectif Libre
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -13,8 +12,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
+import inspect
+import sys
+
 import pbr.version
 
+from keystoneauth1.exceptions import http
 from oslo_utils import timeutils
 
 
@@ -56,3 +59,44 @@ def list_to_cols(list_obj, cols):
     for item in list_obj:
         values.append(dict_to_cols(item, cols))
     return values
+
+
+def http_error_formatter(func):
+    """This decorator catches Http Errors and re-formats them"""
+
+    def wrap(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except http.HttpError as e:
+            raise http.HttpError(message=e.response.text,
+                                 http_status=e.http_status)
+
+    return wrap
+
+
+def format_http_errors(ignore):
+    """Applies ``http_error_formatter`` to all methods of a class.
+
+    :param ignore: List of function names to ignore
+    :type ignore: iterable
+    """
+
+    def wrap(cls):
+        # If you want pretty errors, use python3.
+        # __qualname__ does not exist in python 2
+        if sys.version_info.major < 3:
+            return cls
+
+        def predicate(item):
+            # This avoids decorating functions of parent classes
+            return (inspect.isfunction(item)
+                    and item.__name__ not in ignore
+                    and not item.__name__.startswith('_')
+                    and cls.__name__ in item.__qualname__)
+
+        for name, func in inspect.getmembers(cls, predicate):
+            setattr(cls, name, http_error_formatter(func))
+
+        return cls
+
+    return wrap

cloudkittyclient/v2/client.py

@@ -14,6 +14,8 @@
 #    under the License.
 #
 from cloudkittyclient.v1 import client
+from cloudkittyclient.v2 import dataframes
+from cloudkittyclient.v2.rating import modules
 from cloudkittyclient.v2 import scope
 from cloudkittyclient.v2 import summary
 
@@ -36,5 +38,7 @@ class Client(client.Client):
             **kwargs
         )
 
+        self.dataframes = dataframes.DataframesManager(self.api_client)
         self.scope = scope.ScopeManager(self.api_client)
         self.summary = summary.SummaryManager(self.api_client)
+        self.rating = modules.RatingManager(self.api_client)

cloudkittyclient/v2/dataframes.py

@@ -0,0 +1,77 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import json
+
+from cloudkittyclient.common import base
+from cloudkittyclient import exc
+
+
+class DataframesManager(base.BaseManager):
+    """Class used to handle /v2/dataframes endpoint"""
+
+    url = '/v2/dataframes'
+
+    def add_dataframes(self, **kwargs):
+        """Add DataFrames to the storage backend. Returns nothing.
+
+        :param dataframes: List of dataframes to add to the storage backend.
+        :type dataframes: list of dataframes
+        """
+
+        dataframes = kwargs.get('dataframes')
+
+        if not dataframes:
+            raise exc.ArgumentRequired("'dataframes' argument is required")
+
+        if not isinstance(dataframes, str):
+            try:
+                dataframes = json.dumps(dataframes)
+            except TypeError:
+                raise exc.InvalidArgumentError(
+                    "'dataframes' must be either a string"
+                    "or a JSON serializable object.")
+
+        url = self.get_url(None, kwargs)
+        return self.api_client.post(
+            url,
+            data=dataframes,
+        )
+
+    def get_dataframes(self, **kwargs):
+        """Returns a paginated list of DataFrames.
+
+        This support filters and datetime framing.
+
+        :param offset: Index of the first dataframe that should be returned.
+        :type offset: int
+        :param limit: Maximal number of dataframes to return.
+        :type limit: int
+        :param filters: Optional dict of filters to select data on.
+        :type filters: dict
+        :param begin: Start of the period to gather data from
+        :type begin: datetime.datetime
+        :param end: End of the period to gather data from
+        :type end: datetime.datetime
+        """
+        kwargs['filters'] = ','.join(
+            '{}:{}'.format(k, v) for k, v in
+            (kwargs.get('filters', None) or {}).items()
+        )
+
+        authorized_args = [
+            'offset', 'limit', 'filters', 'begin', 'end']
+
+        url = self.get_url(None, kwargs, authorized_args=authorized_args)
+        return self.api_client.get(url).json()

cloudkittyclient/v2/dataframes_cli.py

@@ -0,0 +1,116 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import argparse
+
+from cliff import command
+from cliff import lister
+from oslo_utils import timeutils
+
+from cloudkittyclient import utils
+
+
+class CliDataframesAdd(command.Command):
+    """Add one or several DataFrame objects to the storage backend."""
+    def get_parser(self, prog_name):
+        parser = super(CliDataframesAdd, self).get_parser(prog_name)
+
+        parser.add_argument(
+            'datafile',
+            type=argparse.FileType('r'),
+            help="File formatted as a JSON object having a DataFrame list"
+                 "under a 'dataframes' key."
+                 "'-' (hyphen) can be specified for using stdin.",
+            )
+
+        return parser
+
+    def take_action(self, parsed_args):
+        with parsed_args.datafile as dfile:
+            dataframes = dfile.read()
+            utils.get_client_from_osc(self).dataframes.add_dataframes(
+                dataframes=dataframes,
+            )
+
+
+class CliDataframesGet(lister.Lister):
+    """Get dataframes from the storage backend."""
+    columns = [
+        ('begin', 'Begin'),
+        ('end', 'End'),
+        ('metric', 'Metric Type'),
+        ('unit', 'Unit'),
+        ('qty', 'Quantity'),
+        ('price', 'Price'),
+        ('groupby', 'Group By'),
+        ('metadata', 'Metadata'),
+    ]
+
+    def get_parser(self, prog_name):
+        parser = super(CliDataframesGet, self).get_parser(prog_name)
+
+        def filter_(elem):
+            if len(elem.split(':')) != 2:
+                raise TypeError
+            return str(elem)
+
+        parser.add_argument('--offset', type=int, default=0,
+                            help='Index of the first dataframe')
+        parser.add_argument('--limit', type=int, default=100,
+                            help='Maximal number of dataframes')
+        parser.add_argument('--filter', type=filter_, action='append',
+                            help="Optional filter, in 'key:value' format. Can "
+                            "be specified several times.")
+        parser.add_argument('-b', '--begin', type=timeutils.parse_isotime,
+                            help="Start of the period to query, in iso8601 "
+                            "format. Example: 2019-05-01T00:00:00Z.")
+        parser.add_argument('-e', '--end', type=timeutils.parse_isotime,
+                            help="End of the period to query, in iso8601 "
+                            "format. Example: 2019-06-01T00:00:00Z.")
+
+        return parser
+
+    def take_action(self, parsed_args):
+        filters = dict(elem.split(':') for elem in (parsed_args.filter or []))
+
+        dataframes = utils.get_client_from_osc(self).dataframes.get_dataframes(
+            offset=parsed_args.offset,
+            limit=parsed_args.limit,
+            begin=parsed_args.begin,
+            end=parsed_args.end,
+            filters=filters,
+        ).get('dataframes', [])
+
+        def format_(d):
+            return ' '.join([
+                '{}="{}"'.format(k, v) for k, v in (d or {}).items()])
+
+        values = []
+        for df in dataframes:
+            period = df['period']
+            usage = df['usage']
+            for metric_type, points in usage.items():
+                for point in points:
+                    values.append([
+                        period['begin'],
+                        period['end'],
+                        metric_type,
+                        point['vol']['unit'],
+                        point['vol']['qty'],
+                        point['rating']['price'],
+                        format_(point.get('groupby', {})),
+                        format_(point.get('metadata', {})),
+                    ])
+
+        return [col[1] for col in self.columns], values

cloudkittyclient/v2/rating/modules.py

@@ -0,0 +1,59 @@
+
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+from cloudkittyclient import exc
+from cloudkittyclient.v1.client import rating
+
+
+class RatingManager(rating.RatingManager):
+    """Class used to handle /v2/rating/modules endpoint"""
+
+    url = '/v2/rating/modules'
+
+    def get_module(self, **kwargs):
+        """Returns the given module.
+
+        If module_id is not specified, returns the list of loaded modules.
+
+        :param module_id: ID of the module on which you want information.
+        :type module_id: str
+        """
+        module_id = kwargs.get('module_id', None)
+        if module_id is not None:
+            url = "{}/{}".format(self.url, module_id)
+        else:
+            url = self.url
+        return self.api_client.get(url).json()
+
+    def update_module(self, **kwargs):
+        """Update the given module.
+
+        :param module_id: Id of the module to update.
+        :type module_id: str
+        :param enabled: Set to True to enable the module, False to disable it.
+        :type enabled: bool
+        :param priority: New priority of the module.
+        :type priority: int
+        """
+        if not kwargs.get('module_id', None):
+            raise exc.ArgumentRequired("'module_id' argument is required.")
+        mutable_fields = ['enabled', 'priority']
+        changes = {}
+        for key, value in kwargs.items():
+            if value is not None and key in mutable_fields:
+                changes[key] = value
+        self.api_client.put("{}/{}".format(self.url, kwargs['module_id']),
+                            json=changes)
+        return self.get_module(**kwargs)

cloudkittyclient/v2/rating/modules_cli.py

@@ -0,0 +1,29 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+from cliff import lister
+
+from cloudkittyclient import utils
+
+
+class CliModuleList(lister.Lister):
+    """Get loaded rating modules list"""
+
+    def get_parser(self, prog_name):
+        parser = super(CliModuleList, self).get_parser(prog_name)
+        return parser
+
+    def take_action(self, parsed_args):
+        resp = utils.get_client_from_osc(self).ratingmodules.get_modules_list()
+        return resp['modules']

doc/requirements.txt

@@ -3,6 +3,7 @@
 # process, which may cause wedges in the gate later.
 
 openstackdocstheme>=1.30.0 # Apache-2.0
-sphinx>=1.6.2,!=1.6.6,!=1.6.7,<2.0.0;python_version=='2.7'  # BSD
-sphinx>=1.6.2,!=1.6.6,!=1.6.7,!=2.1.0;python_version>='3.4' # BSD
+sphinx>=1.8.0,!=2.1.0 # BSD
+sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD
 reno>=2.5.0 # Apache-2.0
+cliff>=2.11.0 # Apache-2.0

doc/source/api_reference/v2/dataframes.rst

@@ -0,0 +1,6 @@
+===========================
+dataframes (/v2/dataframes)
+===========================
+
+.. automodule:: cloudkittyclient.v2.dataframes
+   :members:

doc/source/cli_reference.rst

@@ -5,15 +5,17 @@ CLI Reference
 V1 Client
 =========
 
-.. autoprogram-cliff:: cloudkittyclient.v1
+.. autoprogram-cliff:: cloudkittyclient_v1
    :application: cloudkitty
 
-
 V2 Client
 =========
 
-.. autoprogram-cliff:: cloudkittyclient.v2
+.. autoprogram-cliff:: cloudkittyclient_v2
+   :command: dataframes add
+
+.. autoprogram-cliff:: cloudkittyclient_v2
    :command: scope state get
 
-.. autoprogram-cliff:: cloudkittyclient.v2
+.. autoprogram-cliff:: cloudkittyclient_v2
    :command: summary get

doc/source/conf.py

@@ -23,8 +23,11 @@ extensions = [
     'cliff.sphinxext',
     'sphinx.ext.autodoc',
     'openstackdocstheme',
+    'sphinxcontrib.rsvgconverter',
 ]
 
+autoprogram_cliff_application = 'cloudkitty'
+
 autoprogram_cliff_ignored = [
     "--format", "--column", "--max-width", "--fit-width", "--print-empty",
     "--format-config-file", "--noindent", "--quote", "--sort-column",
@@ -57,7 +60,7 @@ add_function_parentheses = True
 add_module_names = True
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # -- Options for HTML output --------------------------------------------------
 
@@ -74,14 +77,30 @@ html_last_updated_fmt = '%Y-%m-%d %H:%M'
 # Output file base name for HTML help builder.
 htmlhelp_basename = '%sdoc' % project
 
+
+# -- Options for LaTeX output ---------------------------------------------
+
+# Disable usage of xindy https://bugzilla.redhat.com/show_bug.cgi?id=1643664
+latex_use_xindy = False
+
+latex_domain_indices = False
+
+latex_elements = {
+    'makeindex': '',
+    'printindex': '',
+    'preamble': r'\setcounter{tocdepth}{3}',
+}
+
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass
 # [howto/manual]).
+# NOTE: Specify toctree_only=True for a better document structure of
+# the generated PDF file.
 latex_documents = [
     ('index',
-     '%s.tex' % project,
+     'doc-%s.tex' % project,
      u'%s Documentation' % project,
-     u'OpenStack Foundation', 'manual'),
+     u'OpenStack Foundation', 'howto', True),
 ]
 
 # Example configuration for intersphinx: refer to the Python standard library.

lower-constraints.txt

@@ -6,11 +6,11 @@ oslo.utils==3.35 # Apache-2.0
 oslo.log==3.36 # Apache-2.0
 PyYAML==3.12 # MIT
 jsonpath-rw-ext==1.0 # Apache-2.0
-six==1.11 # MIT
 os-client-config==1.29.0 # Apache-2.0
 osc-lib==1.12.1 # Apache-2.0
 
 # test-requirements.txt
+pyflakes==2.1.1
 coverage==4.0 # Apache-2.0
 python-subunit==0.0.18 # Apache-2.0/BSD
 oslotest==1.10.0 # Apache-2.0

releasenotes/notes/add-support-v2-dataframes-baa398fe5ea1b891.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Support for the ``/v2/dataframes`` endpoint has been added to the client.
+    A new ``dataframes add`` CLI command is also available.

releasenotes/notes/add-support-v2-dataframes-be3a17271f3c7188.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Support for the ``GET /v2/dataframes`` endpoint has been added
+    to the client. A new ``dataframes get`` CLI command is also available.

releasenotes/notes/drop-py27-27ea9fb3e40d4987.yaml

@@ -0,0 +1,7 @@
+---
+upgrade:
+  - |
+    Python 2.7 support has been dropped. The last release of
+    ``cloudkittyclient`` to support python 2.7 is OpenStack Train (3.1.0).
+    The minimum version of Python now supported by ``cloudkittyclient``
+    is Python 3.6.

releasenotes/source/conf.py

@@ -82,7 +82,7 @@ exclude_patterns = []
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []

releasenotes/source/index.rst

@@ -8,6 +8,8 @@ Contents
    :maxdepth: 2
 
    unreleased
+   ussuri
+   train
    stein
    rocky
    queens

releasenotes/source/train.rst

@@ -0,0 +1,6 @@
+==========================
+Train Series Release Notes
+==========================
+
+.. release-notes::
+   :branch: stable/train

releasenotes/source/ussuri.rst

@@ -0,0 +1,6 @@
+===========================
+Ussuri Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/ussuri

requirements.txt

@@ -9,6 +9,5 @@ oslo.utils>=3.35 # Apache-2.0
 oslo.log>=3.36 # Apache-2.0
 PyYAML>=3.12 # MIT
 jsonpath-rw-ext>=1.0 # Apache-2.0
-six>=1.11 # MIT
 os-client-config>=1.29.0 # Apache-2.0
 osc-lib>=1.12.1 # Apache-2.0

setup.cfg

@@ -6,6 +6,7 @@ description-file =
 author = OpenStack
 author-email = openstack-discuss@lists.openstack.org
 home-page = https://docs.openstack.org/python-cloudkittyclient/latest/
+python-requires = >=3.6
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology
@@ -13,11 +14,12 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
 
 [files]
 packages =
@@ -86,6 +88,9 @@ openstack.rating.v1 =
     rating_pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript
 
 openstack.rating.v2 =
+    rating_dataframes_get = cloudkittyclient.v2.dataframes_cli:CliDataframesGet
+    rating_dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd
+
     rating_scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet
     rating_scope_state_reset = cloudkittyclient.v2.scope_cli:CliScopeStateReset
 
@@ -135,7 +140,6 @@ openstack.rating.v2 =
     rating_collector_state_get = cloudkittyclient.v1.collector_cli:CliCollectorGetState
     rating_collector_enable = cloudkittyclient.v1.collector_cli:CliCollectorEnable
     rating_collector_disable = cloudkittyclient.v1.collector_cli:CliCollectorDisable
-    rating_dataframes_get = cloudkittyclient.v1.storage_cli:CliGetDataframes
 
     rating_pyscript_create = cloudkittyclient.v1.rating.pyscripts_cli:CliCreateScript
     rating_pyscript_list = cloudkittyclient.v1.rating.pyscripts_cli:CliListScripts
@@ -143,7 +147,7 @@ openstack.rating.v2 =
     rating_pyscript_update = cloudkittyclient.v1.rating.pyscripts_cli:CliUpdateScript
     rating_pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript
 
-cloudkittyclient.v1 =
+cloudkittyclient_v1 =
     total_get = cloudkittyclient.v1.report_cli:CliTotalGet
     summary_get = cloudkittyclient.v1.report_cli:CliSummaryGet
     report_tenant_list = cloudkittyclient.v1.report_cli:CliTenantList
@@ -199,7 +203,10 @@ cloudkittyclient.v1 =
     pyscript_update = cloudkittyclient.v1.rating.pyscripts_cli:CliUpdateScript
     pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript
 
-cloudkittyclient.v2 =
+cloudkittyclient_v2 =
+    dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd
+    dataframes_get = cloudkittyclient.v2.dataframes_cli:CliDataframesGet
+
     scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet
     scope_state_reset = cloudkittyclient.v2.scope_cli:CliScopeStateReset
 
@@ -250,7 +257,6 @@ cloudkittyclient.v2 =
     collector_state_get = cloudkittyclient.v1.collector_cli:CliCollectorGetState
     collector_enable = cloudkittyclient.v1.collector_cli:CliCollectorEnable
     collector_disable = cloudkittyclient.v1.collector_cli:CliCollectorDisable
-    dataframes_get = cloudkittyclient.v1.storage_cli:CliGetDataframes
 
     pyscript_create = cloudkittyclient.v1.rating.pyscripts_cli:CliCreateScript
     pyscript_list = cloudkittyclient.v1.rating.pyscripts_cli:CliListScripts
@@ -263,20 +269,3 @@ keystoneauth1.plugin =
 
 cliff.formatter.list =
     df-to-csv = cloudkittyclient.format:DataframeToCsvFormatter
-
-[upload_sphinx]
-upload-dir = doc/build/html
-
-[compile_catalog]
-directory = cloudkittyclient/locale
-domain = python-cloudkittyclient
-
-[update_catalog]
-domain = python-cloudkittyclient
-output_dir = cloudkittyclient/locale
-input_file = cloudkittyclient/locale/python-cloudkittyclient.pot
-
-[extract_messages]
-keywords = _ gettext ngettext l_ lazy_gettext
-mapping_file = babel.cfg
-output_file = cloudkittyclient/locale/python-cloudkittyclient.pot

setup.py

@@ -13,17 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
 setuptools.setup(
     setup_requires=['pbr>=2.0.0'],
     pbr=True)

test-requirements.txt

@@ -2,11 +2,15 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
+hacking>=3.0,<3.1.0 # Apache-2.0
+
+# remove this pyflakes from here once you bump the
+# hacking to 3.2.0 or above. hacking 3.2.0 takes
+# care of pyflakes version compatibilty.
+pyflakes>=2.1.1
 
 coverage>=4.0,!=4.4 # Apache-2.0
 python-subunit>=0.0.18 # Apache-2.0/BSD
 oslotest>=1.10.0 # Apache-2.0
 stestr>=2.0 # Apache-2.0
-mock>=2.0 # BSD
 python-openstackclient>=3.14 # Apache-2.0

tox.ini

@@ -1,11 +1,13 @@
 [tox]
-minversion = 2.0
-envlist = py27,py36,py37,pypy,pep8
+minversion = 3.1.1
+envlist = py36,py37,pep8
 skipsdist = True
+ignore_basepython_conflict = True
 
 [testenv]
+basepython = python3
 usedevelop = True
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -U {opts} {packages}
+install_command = pip install -U {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
@@ -13,7 +15,6 @@ deps = -r{toxinidir}/requirements.txt
 commands = stestr run {posargs}
 
 [testenv:cover]
-basepython = python3
 setenv =
     VIRTUAL_ENV={envdir}
     PYTHON=coverage run --source cloudkittyclient --parallel-mode
@@ -25,31 +26,42 @@ commands =
     coverage report
 
 [testenv:debug]
-basepython = python3
 commands = oslo_debug_helper -t cloudkittyclient/tests {posargs}
 
-[testenv:functional]
-passenv = OS_CLOUD OS_PROJECT_DOMAIN_ID OS_USER_DOMAIN_ID OS_PROJECT_DOMAIN_NAME OS_USER_DOMAIN_NAME OS_PROJECT_NAME OS_IDENTITY_API_VERSION OS_PASSWORD OS_AUTH_TYPE OS_AUTH_URL OS_USERNAME
-commands = stestr run --concurrency=1 --test-path ./cloudkittyclient/tests/functional
+[testenv:functional-v1]
+passenv = OS_CLOUD OS_PROJECT_DOMAIN_ID OS_USER_DOMAIN_ID OS_PROJECT_DOMAIN_NAME OS_USER_DOMAIN_NAME OS_PROJECT_NAME OS_IDENTITY_API_VERSION OS_PASSWORD OS_AUTH_TYPE OS_AUTH_URL OS_USERNAME OS_ENDPOINT
+setenv = OS_RATING_API_VERSION=1
+commands = stestr run --concurrency=1 --test-path ./cloudkittyclient/tests/functional/v1
+
+[testenv:functional-v2]
+passenv = OS_CLOUD OS_PROJECT_DOMAIN_ID OS_USER_DOMAIN_ID OS_PROJECT_DOMAIN_NAME OS_USER_DOMAIN_NAME OS_PROJECT_NAME OS_IDENTITY_API_VERSION OS_PASSWORD OS_AUTH_TYPE OS_AUTH_URL OS_USERNAME OS_ENDPOINT
+setenv = OS_RATING_API_VERSION=2
+commands = stestr run --concurrency=1 --test-path ./cloudkittyclient/tests/functional/v2
 
 [testenv:pep8]
-basepython = python3
 commands = flake8
 
 [testenv:venv]
-basepython = python3
 commands = {posargs}
 
 [testenv:docs]
-basepython = python3
 deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -W -b html doc/source doc/build/html
+commands = sphinx-build --keep-going -b html doc/source doc/build/html
+
+[testenv:pdf-docs]
+envdir = {toxworkdir}/docs
+deps = {[testenv:docs]deps}
+whitelist_externals =
+  make
+commands =
+  sphinx-build --keep-going -b latex doc/source doc/build/pdf
+  make -C doc/build/pdf
 
 [flake8]
 # E123, E125 skipped as they are invalid PEP-8.
 
 show-source = True
-ignore = E123,E125
+ignore = E123,E125,W503,W504
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,releasenotes
 
@@ -57,8 +69,14 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,releasenotes
 import_exceptions = cloudkittyclient.i18n
 
 [testenv:releasenotes]
-basepython = python3
 deps =
   -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
   -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -a -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+commands =
+  sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
+
+[testenv:lower-constraints]
+deps =
+  -c{toxinidir}/lower-constraints.txt
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/requirements.txt