Update TOX_CONSTRAINTS_FILE for stable/victoria

Update the URL to the upper-constraints file to point to the redirect
rule on releases.openstack.org so that anyone working on this branch
will switch to the correct upper-constraints list automatically when
the requirements repository branches.

Until the requirements repository has as stable/victoria branch, tests will
continue to use the upper-constraints list on master.

Change-Id: I6cfaedc6183cf3b29608cf90d2a1b175d4550cb2

.gitreview

@@ -2,3 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/python-cloudkittyclient.git
+defaultbranch=stable/victoria

.zuul.yaml

@@ -46,7 +46,7 @@
       - openstack-lower-constraints-jobs
       - check-requirements
       - openstack-cover-jobs
-      - openstack-python3-ussuri-jobs
+      - openstack-python3-victoria-jobs
       - openstackclient-plugin-jobs
       - publish-openstack-docs-pti
     check:

babel.cfg

@@ -1,2 +0,0 @@
-[python: **.py]
-

cloudkittyclient/common/base.py

@@ -12,11 +12,9 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-#
-from string import Formatter as StringFormatter
 
-from six import add_metaclass
-from six.moves.urllib.parse import urlencode
+from string import Formatter as StringFormatter
+from urllib.parse import urlencode
 
 from cloudkittyclient import utils
 
@@ -31,8 +29,7 @@ class HttpDecoratorMeta(type):
         )
 
 
-@add_metaclass(HttpDecoratorMeta)
-class BaseManager(object):
+class BaseManager(object, metaclass=HttpDecoratorMeta):
     """Base class for Endpoint Manager objects."""
 
     url = ''

cloudkittyclient/tests/functional/v2/test_dataframes.py

@@ -162,6 +162,11 @@ class CkDataframesTest(base.BaseFunctionalTest):
                 has_output=False,
             )
 
+    def test_dataframes_get(self):
+        # TODO(jferrieu): functional tests will be added in another
+        # patch for `dataframes get`
+        pass
+
 
 class OSCDataframesTest(CkDataframesTest):
     def __init__(self, *args, **kwargs):

cloudkittyclient/tests/unit/v1/test_hashmap.py

@@ -13,7 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-import mock
+from unittest import mock
 
 from cloudkittyclient import exc
 from cloudkittyclient.tests.unit.v1 import base

cloudkittyclient/tests/unit/v1/test_report_cli.py

@@ -13,7 +13,7 @@
 #    under the License.
 #
 import collections
-import mock
+from unittest import mock
 
 from cloudkittyclient.tests.unit.v1 import base
 from cloudkittyclient.v1 import report_cli

cloudkittyclient/tests/unit/v2/test_dataframes.py

@@ -14,6 +14,8 @@
 #
 import json
 
+from collections import OrderedDict
+
 from cloudkittyclient import exc
 from cloudkittyclient.tests.unit.v2 import base
 
@@ -149,3 +151,22 @@ class TestDataframes(base.BaseAPIEndpointTestCase):
         self.assertRaises(
             exc.ArgumentRequired,
             self.dataframes.add_dataframes)
+
+    def test_get_dataframes(self):
+        self.dataframes.get_dataframes()
+        self.api_client.get.assert_called_once_with('/v2/dataframes')
+
+    def test_get_dataframes_with_pagination_args(self):
+        self.dataframes.get_dataframes(offset=10, limit=10)
+        try:
+            self.api_client.get.assert_called_once_with(
+                '/v2/dataframes?limit=10&offset=10')
+        except AssertionError:
+            self.api_client.get.assert_called_once_with(
+                '/v2/dataframes?offset=10&limit=10')
+
+    def test_get_dataframes_filters(self):
+        self.dataframes.get_dataframes(
+            filters=OrderedDict([('one', 'two'), ('three', 'four')]))
+        self.api_client.get.assert_called_once_with(
+            '/v2/dataframes?filters=one%3Atwo%2Cthree%3Afour')

cloudkittyclient/tests/utils.py

@@ -12,12 +12,13 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+from unittest import mock
+
 import fixtures
 import testtools
 
 from keystoneauth1 import adapter
 from keystoneauth1 import session
-import mock
 
 
 class BaseTestCase(testtools.TestCase):

cloudkittyclient/v2/dataframes.py

@@ -13,7 +13,6 @@
 #    under the License.
 #
 import json
-import six
 
 from cloudkittyclient.common import base
 from cloudkittyclient import exc
@@ -36,7 +35,7 @@ class DataframesManager(base.BaseManager):
         if not dataframes:
             raise exc.ArgumentRequired("'dataframes' argument is required")
 
-        if not isinstance(dataframes, six.string_types):
+        if not isinstance(dataframes, str):
             try:
                 dataframes = json.dumps(dataframes)
             except TypeError:
@@ -49,3 +48,30 @@ class DataframesManager(base.BaseManager):
             url,
             data=dataframes,
         )
+
+    def get_dataframes(self, **kwargs):
+        """Returns a paginated list of DataFrames.
+
+        This support filters and datetime framing.
+
+        :param offset: Index of the first dataframe that should be returned.
+        :type offset: int
+        :param limit: Maximal number of dataframes to return.
+        :type limit: int
+        :param filters: Optional dict of filters to select data on.
+        :type filters: dict
+        :param begin: Start of the period to gather data from
+        :type begin: datetime.datetime
+        :param end: End of the period to gather data from
+        :type end: datetime.datetime
+        """
+        kwargs['filters'] = ','.join(
+            '{}:{}'.format(k, v) for k, v in
+            (kwargs.get('filters', None) or {}).items()
+        )
+
+        authorized_args = [
+            'offset', 'limit', 'filters', 'begin', 'end']
+
+        url = self.get_url(None, kwargs, authorized_args=authorized_args)
+        return self.api_client.get(url).json()

cloudkittyclient/v2/dataframes_cli.py

@@ -15,6 +15,8 @@
 import argparse
 
 from cliff import command
+from cliff import lister
+from oslo_utils import timeutils
 
 from cloudkittyclient import utils
 
@@ -40,3 +42,75 @@ class CliDataframesAdd(command.Command):
             utils.get_client_from_osc(self).dataframes.add_dataframes(
                 dataframes=dataframes,
             )
+
+
+class CliDataframesGet(lister.Lister):
+    """Get dataframes from the storage backend."""
+    columns = [
+        ('begin', 'Begin'),
+        ('end', 'End'),
+        ('metric', 'Metric Type'),
+        ('unit', 'Unit'),
+        ('qty', 'Quantity'),
+        ('price', 'Price'),
+        ('groupby', 'Group By'),
+        ('metadata', 'Metadata'),
+    ]
+
+    def get_parser(self, prog_name):
+        parser = super(CliDataframesGet, self).get_parser(prog_name)
+
+        def filter_(elem):
+            if len(elem.split(':')) != 2:
+                raise TypeError
+            return str(elem)
+
+        parser.add_argument('--offset', type=int, default=0,
+                            help='Index of the first dataframe')
+        parser.add_argument('--limit', type=int, default=100,
+                            help='Maximal number of dataframes')
+        parser.add_argument('--filter', type=filter_, action='append',
+                            help="Optional filter, in 'key:value' format. Can "
+                            "be specified several times.")
+        parser.add_argument('-b', '--begin', type=timeutils.parse_isotime,
+                            help="Start of the period to query, in iso8601 "
+                            "format. Example: 2019-05-01T00:00:00Z.")
+        parser.add_argument('-e', '--end', type=timeutils.parse_isotime,
+                            help="End of the period to query, in iso8601 "
+                            "format. Example: 2019-06-01T00:00:00Z.")
+
+        return parser
+
+    def take_action(self, parsed_args):
+        filters = dict(elem.split(':') for elem in (parsed_args.filter or []))
+
+        dataframes = utils.get_client_from_osc(self).dataframes.get_dataframes(
+            offset=parsed_args.offset,
+            limit=parsed_args.limit,
+            begin=parsed_args.begin,
+            end=parsed_args.end,
+            filters=filters,
+        ).get('dataframes', [])
+
+        def format_(d):
+            return ' '.join([
+                '{}="{}"'.format(k, v) for k, v in (d or {}).items()])
+
+        values = []
+        for df in dataframes:
+            period = df['period']
+            usage = df['usage']
+            for metric_type, points in usage.items():
+                for point in points:
+                    values.append([
+                        period['begin'],
+                        period['end'],
+                        metric_type,
+                        point['vol']['unit'],
+                        point['vol']['qty'],
+                        point['rating']['price'],
+                        format_(point.get('groupby', {})),
+                        format_(point.get('metadata', {})),
+                    ])
+
+        return [col[1] for col in self.columns], values

doc/source/conf.py

@@ -60,7 +60,7 @@ add_function_parentheses = True
 add_module_names = True
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # -- Options for HTML output --------------------------------------------------
 

lower-constraints.txt

@@ -6,11 +6,11 @@ oslo.utils==3.35 # Apache-2.0
 oslo.log==3.36 # Apache-2.0
 PyYAML==3.12 # MIT
 jsonpath-rw-ext==1.0 # Apache-2.0
-six==1.11 # MIT
 os-client-config==1.29.0 # Apache-2.0
 osc-lib==1.12.1 # Apache-2.0
 
 # test-requirements.txt
+pyflakes==2.1.1
 coverage==4.0 # Apache-2.0
 python-subunit==0.0.18 # Apache-2.0/BSD
 oslotest==1.10.0 # Apache-2.0

releasenotes/notes/add-support-v2-dataframes-be3a17271f3c7188.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Support for the ``GET /v2/dataframes`` endpoint has been added
+    to the client. A new ``dataframes get`` CLI command is also available.

releasenotes/source/conf.py

@@ -82,7 +82,7 @@ exclude_patterns = []
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'native'
 
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []

releasenotes/source/index.rst

@@ -8,6 +8,7 @@ Contents
    :maxdepth: 2
 
    unreleased
+   ussuri
    train
    stein
    rocky

releasenotes/source/ussuri.rst

@@ -0,0 +1,6 @@
+===========================
+Ussuri Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/ussuri

requirements.txt

@@ -9,6 +9,5 @@ oslo.utils>=3.35 # Apache-2.0
 oslo.log>=3.36 # Apache-2.0
 PyYAML>=3.12 # MIT
 jsonpath-rw-ext>=1.0 # Apache-2.0
-six>=1.11 # MIT
 os-client-config>=1.29.0 # Apache-2.0
 osc-lib>=1.12.1 # Apache-2.0

setup.cfg

@@ -14,9 +14,12 @@ classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
     Programming Language :: Python
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
 
 [files]
 packages =
@@ -85,6 +88,7 @@ openstack.rating.v1 =
     rating_pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript
 
 openstack.rating.v2 =
+    rating_dataframes_get = cloudkittyclient.v2.dataframes_cli:CliDataframesGet
     rating_dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd
 
     rating_scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet
@@ -136,7 +140,6 @@ openstack.rating.v2 =
     rating_collector_state_get = cloudkittyclient.v1.collector_cli:CliCollectorGetState
     rating_collector_enable = cloudkittyclient.v1.collector_cli:CliCollectorEnable
     rating_collector_disable = cloudkittyclient.v1.collector_cli:CliCollectorDisable
-    rating_dataframes_get = cloudkittyclient.v1.storage_cli:CliGetDataframes
 
     rating_pyscript_create = cloudkittyclient.v1.rating.pyscripts_cli:CliCreateScript
     rating_pyscript_list = cloudkittyclient.v1.rating.pyscripts_cli:CliListScripts
@@ -202,6 +205,7 @@ cloudkittyclient_v1 =
 
 cloudkittyclient_v2 =
     dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd
+    dataframes_get = cloudkittyclient.v2.dataframes_cli:CliDataframesGet
 
     scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet
     scope_state_reset = cloudkittyclient.v2.scope_cli:CliScopeStateReset
@@ -253,7 +257,6 @@ cloudkittyclient_v2 =
     collector_state_get = cloudkittyclient.v1.collector_cli:CliCollectorGetState
     collector_enable = cloudkittyclient.v1.collector_cli:CliCollectorEnable
     collector_disable = cloudkittyclient.v1.collector_cli:CliCollectorDisable
-    dataframes_get = cloudkittyclient.v1.storage_cli:CliGetDataframes
 
     pyscript_create = cloudkittyclient.v1.rating.pyscripts_cli:CliCreateScript
     pyscript_list = cloudkittyclient.v1.rating.pyscripts_cli:CliListScripts
@@ -266,20 +269,3 @@ keystoneauth1.plugin =
 
 cliff.formatter.list =
     df-to-csv = cloudkittyclient.format:DataframeToCsvFormatter
-
-[upload_sphinx]
-upload-dir = doc/build/html
-
-[compile_catalog]
-directory = cloudkittyclient/locale
-domain = python-cloudkittyclient
-
-[update_catalog]
-domain = python-cloudkittyclient
-output_dir = cloudkittyclient/locale
-input_file = cloudkittyclient/locale/python-cloudkittyclient.pot
-
-[extract_messages]
-keywords = _ gettext ngettext l_ lazy_gettext
-mapping_file = babel.cfg
-output_file = cloudkittyclient/locale/python-cloudkittyclient.pot

setup.py

@@ -13,17 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
-# In python < 2.7.4, a lazy loading of package `pbr` will break
-# setuptools if some other modules registered functions in `atexit`.
-# solution from: http://bugs.python.org/issue15881#msg170215
-try:
-    import multiprocessing  # noqa
-except ImportError:
-    pass
-
 setuptools.setup(
     setup_requires=['pbr>=2.0.0'],
     pbr=True)

test-requirements.txt

@@ -2,11 +2,15 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
+hacking>=3.0,<3.1.0 # Apache-2.0
+
+# remove this pyflakes from here once you bump the
+# hacking to 3.2.0 or above. hacking 3.2.0 takes
+# care of pyflakes version compatibilty.
+pyflakes>=2.1.1
 
 coverage>=4.0,!=4.4 # Apache-2.0
 python-subunit>=0.0.18 # Apache-2.0/BSD
 oslotest>=1.10.0 # Apache-2.0
 stestr>=2.0 # Apache-2.0
-mock>=2.0 # BSD
 python-openstackclient>=3.14 # Apache-2.0

tox.ini

@@ -61,7 +61,7 @@ commands =
 # E123, E125 skipped as they are invalid PEP-8.
 
 show-source = True
-ignore = E123,E125
+ignore = E123,E125,W503,W504
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,releasenotes
 
@@ -70,7 +70,7 @@ import_exceptions = cloudkittyclient.i18n
 
 [testenv:releasenotes]
 deps =
-  -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt}
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/victoria}
   -r{toxinidir}/doc/requirements.txt
 commands =
   sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html