Merge "Add support for POST /v2/dataframes API endpoint to the client" into stable/train

Support for the ``/v2/dataframes`` endpoint has been added to the client.
A new ``dataframes add`` CLI command is also available.

Change-Id: I7fe9072d7280f251edc865a653a0b9ed2ab26c90
Story: 2005890
Task: 35970
(cherry picked from commit c8d7a9e1c5)

.gitreview

@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/python-cloudkittyclient.git
+defaultbranch=stable/train

.zuul.yaml

@@ -1,5 +1,5 @@
 - job:
-    name: cloudkittyclient-devstack-functional
+    name: cloudkittyclient-devstack-functional-base
     parent: devstack
     description: |
       Job for cloudkittyclient functional tests
@@ -26,11 +26,22 @@
         horizon: false
       tox_install_siblings: false
       zuul_work_dir: src/opendev.org/openstack/python-cloudkittyclient
-      tox_envlist: functional
 
 - job:
-    name: cloudkittyclient-devstack-functional-py3
-    parent: cloudkittyclient-devstack-functional
+    name: cloudkittyclient-devstack-functional-v1-client
+    parent: cloudkittyclient-devstack-functional-base
+    vars:
+      tox_envlist: functional-v1
+
+- job:
+    name: cloudkittyclient-devstack-functional-v2-client
+    parent: cloudkittyclient-devstack-functional-base
+    vars:
+      tox_envlist: functional-v2
+
+- job:
+    name: cloudkittyclient-devstack-functional-base-py3
+    parent: cloudkittyclient-devstack-functional-base
     description: |
       Job for cloudkittyclient functional tests, ran in python3.
     vars:
@@ -38,8 +49,21 @@
         DEVSTACK_GATE_USE_PYTHON3: "True"
         USE_PYTHON3: "True"
 
+- job:
+    name: cloudkittyclient-devstack-functional-v1-client-py3
+    parent: cloudkittyclient-devstack-functional-base-py3
+    vars:
+      tox_envlist: functional-v1
+
+- job:
+    name: cloudkittyclient-devstack-functional-v2-client-py3
+    parent: cloudkittyclient-devstack-functional-base-py3
+    vars:
+      tox_envlist: functional-v2
+
 - project:
     templates:
+      - openstack-lower-constraints-jobs
       - check-requirements
       - openstack-cover-jobs
       - openstack-python-jobs
@@ -48,13 +72,21 @@
       - publish-openstack-docs-pti
     check:
       jobs:
-        - cloudkittyclient-devstack-functional:
+        - cloudkittyclient-devstack-functional-v1-client:
             voting: true
-        - cloudkittyclient-devstack-functional-py3:
+        - cloudkittyclient-devstack-functional-v2-client:
+            voting: true
+        - cloudkittyclient-devstack-functional-v1-client-py3:
+            voting: true
+        - cloudkittyclient-devstack-functional-v2-client-py3:
             voting: true
     gate:
       jobs:
-        - cloudkittyclient-devstack-functional:
+        - cloudkittyclient-devstack-functional-v1-client:
             voting: true
-        - cloudkittyclient-devstack-functional-py3:
+        - cloudkittyclient-devstack-functional-v2-client:
+            voting: true
+        - cloudkittyclient-devstack-functional-v1-client-py3:
+            voting: true
+        - cloudkittyclient-devstack-functional-v2-client-py3:
             voting: true

cloudkittyclient/common/client.py

@@ -26,6 +26,9 @@ class BaseClient(object):
                  insecure=False,
                  **kwargs):
         adapter_options.setdefault('service_type', 'rating')
+        adapter_options.setdefault('additional_headers', {
+            'Content-Type': 'application/json',
+        })
 
         if insecure:
             verify_cert = False

cloudkittyclient/tests/functional/base.py

@@ -24,24 +24,30 @@ from cloudkittyclient.tests import utils
 class BaseFunctionalTest(utils.BaseTestCase):
 
     def _run(self, executable, action,
-             flags='', params='', fmt='-f json', has_output=True):
+             flags='', params='', fmt='-f json', stdin=None, has_output=True):
         if not has_output:
             fmt = ''
         cmd = ' '.join([executable, flags, action, params, fmt])
         cmd = shlex.split(cmd)
-        p = subprocess.Popen(cmd, env=os.environ.copy(), shell=False,
-                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
+        p = subprocess.Popen(
+            cmd, env=os.environ.copy(), shell=False,
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE if stdin else None,
+        )
+        stdout, stderr = p.communicate(input=stdin)
         if p.returncode != 0:
             raise RuntimeError('"{cmd}" returned {val}: {msg}'.format(
                 cmd=' '.join(cmd), val=p.returncode, msg=stderr))
         return json.loads(stdout) if has_output else None
 
     def openstack(self, action,
-                  flags='', params='', fmt='-f json', has_output=True):
+                  flags='', params='', fmt='-f json',
+                  stdin=None, has_output=True):
         return self._run('openstack rating', action,
-                         flags, params, fmt, has_output)
+                         flags, params, fmt, stdin, has_output)
 
     def cloudkitty(self, action,
-                   flags='', params='', fmt='-f json', has_output=True):
-        return self._run('cloudkitty', action, flags, params, fmt, has_output)
+                   flags='', params='', fmt='-f json',
+                   stdin=None, has_output=True):
+        return self._run('cloudkitty', action, flags, params, fmt,
+                         stdin, has_output)

cloudkittyclient/tests/functional/v2/test_dataframes.py

@@ -0,0 +1,169 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import os
+import uuid
+
+from cloudkittyclient.tests.functional import base
+
+
+class CkDataframesTest(base.BaseFunctionalTest):
+    dataframes_data = """
+    {
+        "dataframes": [
+            {
+                "period": {
+                    "begin": "20190723T122810Z",
+                    "end": "20190723T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 1.2
+                            },
+                            "rating": {
+                                "price": 0.04
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 200.4
+                            },
+                            "rating": {
+                                "price": 0.06
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            },
+            {
+                "period": {
+                    "begin": "20190823T122810Z",
+                    "end": "20190823T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 2.4
+                            },
+                            "rating": {
+                                "price": 0.08
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 400.8
+                            },
+                            "rating": {
+                                "price": 0.12
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(CkDataframesTest, self).__init__(*args, **kwargs)
+        self.runner = self.cloudkitty
+
+    def setUp(self):
+        super(CkDataframesTest, self).setUp()
+
+        self.fixture_file_name = '{}.json'.format(uuid.uuid4())
+        with open(self.fixture_file_name, 'w') as f:
+            f.write(self.dataframes_data)
+
+    def tearDown(self):
+        files = os.listdir('.')
+        if self.fixture_file_name in files:
+            os.remove(self.fixture_file_name)
+
+        super(CkDataframesTest, self).tearDown()
+
+    def test_dataframes_add_with_no_args(self):
+        self.assertRaisesRegexp(
+            RuntimeError,
+            'error: too few arguments',
+            self.runner,
+            'dataframes add',
+            fmt='',
+            has_output=False,
+        )
+
+    def test_dataframes_add(self):
+        self.runner(
+            'dataframes add {}'.format(self.fixture_file_name),
+            fmt='',
+            has_output=False,
+        )
+
+    def test_dataframes_add_with_hyphen_stdin(self):
+        with open(self.fixture_file_name, 'r') as f:
+            self.runner(
+                'dataframes add -',
+                fmt='',
+                stdin=f.read().encode(),
+                has_output=False,
+            )
+
+
+class OSCDataframesTest(CkDataframesTest):
+    def __init__(self, *args, **kwargs):
+        super(OSCDataframesTest, self).__init__(*args, **kwargs)
+        self.runner = self.openstack

cloudkittyclient/tests/unit/v2/base.py

@@ -13,6 +13,7 @@
 #    under the License.
 
 from cloudkittyclient.tests import utils
+from cloudkittyclient.v2 import dataframes
 from cloudkittyclient.v2 import scope
 from cloudkittyclient.v2 import summary
 
@@ -22,5 +23,6 @@ class BaseAPIEndpointTestCase(utils.BaseTestCase):
     def setUp(self):
         super(BaseAPIEndpointTestCase, self).setUp()
         self.api_client = utils.FakeHTTPClient()
+        self.dataframes = dataframes.DataframesManager(self.api_client)
         self.scope = scope.ScopeManager(self.api_client)
         self.summary = summary.SummaryManager(self.api_client)

cloudkittyclient/tests/unit/v2/test_dataframes.py

@@ -0,0 +1,151 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import json
+
+from cloudkittyclient import exc
+from cloudkittyclient.tests.unit.v2 import base
+
+
+class TestDataframes(base.BaseAPIEndpointTestCase):
+    dataframes_data = """
+    {
+        "dataframes": [
+            {
+                "period": {
+                    "begin": "20190723T122810Z",
+                    "end": "20190723T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 1.2
+                            },
+                            "rating": {
+                                "price": 0.04
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 200.4
+                            },
+                            "rating": {
+                                "price": 0.06
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            },
+            {
+                "period": {
+                    "begin": "20190823T122810Z",
+                    "end": "20190823T132810Z"
+                },
+                "usage": {
+                    "metric_one": [
+                        {
+                            "vol": {
+                                "unit": "GiB",
+                                "qty": 2.4
+                            },
+                            "rating": {
+                                "price": 0.08
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ],
+                    "metric_two": [
+                        {
+                            "vol": {
+                                "unit": "MB",
+                                "qty": 400.8
+                            },
+                            "rating": {
+                                "price": 0.12
+                            },
+                            "groupby": {
+                                "group_one": "one",
+                                "group_two": "two"
+                            },
+                            "metadata": {
+                                "attr_one": "one",
+                                "attr_two": "two"
+                            }
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+    """
+
+    def test_add_dataframes_with_string(self):
+        self.dataframes.add_dataframes(
+            dataframes=self.dataframes_data,
+        )
+        self.api_client.post.assert_called_once_with(
+            '/v2/dataframes',
+            data=self.dataframes_data,
+        )
+
+    def test_add_dataframes_with_json_object(self):
+        json_data = json.loads(self.dataframes_data)
+
+        self.dataframes.add_dataframes(
+            dataframes=json_data,
+        )
+        self.api_client.post.assert_called_once_with(
+            '/v2/dataframes',
+            data=json.dumps(json_data),
+        )
+
+    def test_add_dataframes_with_neither_string_nor_object_raises_exc(self):
+        self.assertRaises(
+            exc.InvalidArgumentError,
+            self.dataframes.add_dataframes,
+            dataframes=[open],
+        )
+
+    def test_add_dataframes_with_no_args_raises_exc(self):
+        self.assertRaises(
+            exc.ArgumentRequired,
+            self.dataframes.add_dataframes)

cloudkittyclient/v2/client.py

@@ -14,6 +14,7 @@
 #    under the License.
 #
 from cloudkittyclient.v1 import client
+from cloudkittyclient.v2 import dataframes
 from cloudkittyclient.v2 import scope
 from cloudkittyclient.v2 import summary
 
@@ -36,5 +37,6 @@ class Client(client.Client):
             **kwargs
         )
 
+        self.dataframes = dataframes.DataframesManager(self.api_client)
         self.scope = scope.ScopeManager(self.api_client)
         self.summary = summary.SummaryManager(self.api_client)

cloudkittyclient/v2/dataframes.py

@@ -0,0 +1,51 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import json
+import six
+
+from cloudkittyclient.common import base
+from cloudkittyclient import exc
+
+
+class DataframesManager(base.BaseManager):
+    """Class used to handle /v2/dataframes endpoint"""
+
+    url = '/v2/dataframes'
+
+    def add_dataframes(self, **kwargs):
+        """Add DataFrames to the storage backend. Returns nothing.
+
+        :param dataframes: List of dataframes to add to the storage backend.
+        :type dataframes: list of dataframes
+        """
+
+        dataframes = kwargs.get('dataframes')
+
+        if not dataframes:
+            raise exc.ArgumentRequired("'dataframes' argument is required")
+
+        if not isinstance(dataframes, six.string_types):
+            try:
+                dataframes = json.dumps(dataframes)
+            except TypeError:
+                raise exc.InvalidArgumentError(
+                    "'dataframes' must be either a string"
+                    "or a JSON serializable object.")
+
+        url = self.get_url(None, kwargs)
+        return self.api_client.post(
+            url,
+            data=dataframes,
+        )

cloudkittyclient/v2/dataframes_cli.py

@@ -0,0 +1,42 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import argparse
+
+from cliff import command
+
+from cloudkittyclient import utils
+
+
+class CliDataframesAdd(command.Command):
+    """Add one or several DataFrame objects to the storage backend."""
+    def get_parser(self, prog_name):
+        parser = super(CliDataframesAdd, self).get_parser(prog_name)
+
+        parser.add_argument(
+            'datafile',
+            type=argparse.FileType('r'),
+            help="File formatted as a JSON object having a DataFrame list"
+                 "under a 'dataframes' key."
+                 "'-' (hyphen) can be specified for using stdin.",
+            )
+
+        return parser
+
+    def take_action(self, parsed_args):
+        with parsed_args.datafile as dfile:
+            dataframes = dfile.read()
+            utils.get_client_from_osc(self).dataframes.add_dataframes(
+                dataframes=dataframes,
+            )

doc/source/api_reference/v2/dataframes.rst

@@ -0,0 +1,6 @@
+===========================
+dataframes (/v2/dataframes)
+===========================
+
+.. automodule:: cloudkittyclient.v2.dataframes
+   :members:

doc/source/cli_reference.rst

@@ -12,6 +12,9 @@ V1 Client
 V2 Client
 =========
 
+.. autoprogram-cliff:: cloudkittyclient.v2
+   :command: dataframes add
+
 .. autoprogram-cliff:: cloudkittyclient.v2
    :command: scope state get
 

releasenotes/notes/add-support-v2-dataframes-baa398fe5ea1b891.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Support for the ``/v2/dataframes`` endpoint has been added to the client.
+    A new ``dataframes add`` CLI command is also available.

setup.cfg

@@ -86,6 +86,8 @@ openstack.rating.v1 =
     rating_pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript
 
 openstack.rating.v2 =
+    rating_dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd
+
     rating_scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet
     rating_scope_state_reset = cloudkittyclient.v2.scope_cli:CliScopeStateReset
 
@@ -200,6 +202,8 @@ cloudkittyclient.v1 =
     pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript
 
 cloudkittyclient.v2 =
+    dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd
+
     scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet
     scope_state_reset = cloudkittyclient.v2.scope_cli:CliScopeStateReset
 

tox.ini

@@ -5,7 +5,7 @@ skipsdist = True
 
 [testenv]
 usedevelop = True
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -U {opts} {packages}
+install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train} -U {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
@@ -28,9 +28,15 @@ commands =
 basepython = python3
 commands = oslo_debug_helper -t cloudkittyclient/tests {posargs}
 
-[testenv:functional]
-passenv = OS_CLOUD OS_PROJECT_DOMAIN_ID OS_USER_DOMAIN_ID OS_PROJECT_DOMAIN_NAME OS_USER_DOMAIN_NAME OS_PROJECT_NAME OS_IDENTITY_API_VERSION OS_PASSWORD OS_AUTH_TYPE OS_AUTH_URL OS_USERNAME
-commands = stestr run --concurrency=1 --test-path ./cloudkittyclient/tests/functional
+[testenv:functional-v1]
+passenv = OS_CLOUD OS_PROJECT_DOMAIN_ID OS_USER_DOMAIN_ID OS_PROJECT_DOMAIN_NAME OS_USER_DOMAIN_NAME OS_PROJECT_NAME OS_IDENTITY_API_VERSION OS_PASSWORD OS_AUTH_TYPE OS_AUTH_URL OS_USERNAME OS_ENDPOINT
+setenv = OS_RATING_API_VERSION=1
+commands = stestr run --concurrency=1 --test-path ./cloudkittyclient/tests/functional/v1
+
+[testenv:functional-v2]
+passenv = OS_CLOUD OS_PROJECT_DOMAIN_ID OS_USER_DOMAIN_ID OS_PROJECT_DOMAIN_NAME OS_USER_DOMAIN_NAME OS_PROJECT_NAME OS_IDENTITY_API_VERSION OS_PASSWORD OS_AUTH_TYPE OS_AUTH_URL OS_USERNAME OS_ENDPOINT
+setenv = OS_RATING_API_VERSION=2
+commands = stestr run --concurrency=1 --test-path ./cloudkittyclient/tests/functional/v2
 
 [testenv:pep8]
 basepython = python3
@@ -59,6 +65,13 @@ import_exceptions = cloudkittyclient.i18n
 [testenv:releasenotes]
 basepython = python3
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
+  -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/train}
   -r{toxinidir}/doc/requirements.txt
 commands = sphinx-build -a -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+
+[testenv:lower-constraints]
+basepython = python3
+deps =
+  -c{toxinidir}/lower-constraints.txt
+  -r{toxinidir}/test-requirements.txt
+  -r{toxinidir}/requirements.txt