X Tutup
Skip to content

Commit 05d5bdc

Browse files
JohnVillalovosJohnVillalovos
committed
chore: enable using GitLab EE in functional tests
Enable using GitLab Enterprise Edition (EE) in the functional tests. This will allow us to add functional tests for EE only features in the functional tests.
1 parent f26bf7d commit 05d5bdc

File tree

8 files changed

+222
-49
lines changed

8 files changed

+222
-49
lines changed

‎pyproject.toml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,10 @@ disable = [
8787
"useless-object-inheritance",
8888

8989
]
90+
91+
[tool.pytest.ini_options]
92+
log_cli = true
93+
log_cli_level = "INFO"
94+
log_cli_format = "%(asctime)s.%(msecs)03d [%(levelname)8s] (%(filename)s:%(funcName)s:L%(lineno)s) %(message)s"
95+
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
96+
addopts = "--color=yes"

‎tests/functional/conftest.py‎

Lines changed: 130 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import logging
12
import tempfile
23
import time
34
import uuid
@@ -10,57 +11,14 @@
1011
import gitlab.base
1112

1213
SLEEP_INTERVAL = 0.1
13-
TIMEOUT = 60 # seconds before timeout will occur
14+
TIMEOUT = 3 * 60 # seconds before timeout will occur
1415

1516

1617
@pytest.fixture(scope="session")
1718
def fixture_dir(test_dir):
1819
return test_dir / "functional" / "fixtures"
1920

2021

21-
def reset_gitlab(gl):
22-
# previously tools/reset_gitlab.py
23-
for project in gl.projects.list():
24-
for deploy_token in project.deploytokens.list():
25-
deploy_token.delete()
26-
project.delete()
27-
for group in gl.groups.list():
28-
for deploy_token in group.deploytokens.list():
29-
deploy_token.delete()
30-
group.delete()
31-
for variable in gl.variables.list():
32-
variable.delete()
33-
for user in gl.users.list():
34-
if user.username != "root":
35-
user.delete(hard_delete=True)
36-
37-
max_iterations = int(TIMEOUT / SLEEP_INTERVAL)
38-
39-
# Ensure everything has been reset
40-
start_time = time.perf_counter()
41-
42-
def wait_for_maximum_list_length(
43-
rest_manager: gitlab.base.RESTManager, description: str, max_length: int = 0
44-
) -> None:
45-
"""Wait for the list() length to be no greater than expected maximum or fail
46-
test if timeout is exceeded"""
47-
for _ in range(max_iterations):
48-
if len(rest_manager.list()) <= max_length:
49-
break
50-
time.sleep(SLEEP_INTERVAL)
51-
assert len(rest_manager.list()) <= max_length, (
52-
f"Did not delete required items for {description}. "
53-
f"Elapsed_time: {time.perf_counter() - start_time}"
54-
)
55-
56-
wait_for_maximum_list_length(rest_manager=gl.projects, description="projects")
57-
wait_for_maximum_list_length(rest_manager=gl.groups, description="groups")
58-
wait_for_maximum_list_length(rest_manager=gl.variables, description="variables")
59-
wait_for_maximum_list_length(
60-
rest_manager=gl.users, description="users", max_length=1
61-
)
62-
63-
6422
def set_token(container, fixture_dir):
6523
set_token_rb = fixture_dir / "set_token.rb"
6624

@@ -163,9 +121,11 @@ def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_
163121
config_file = temp_dir / "python-gitlab.cfg"
164122
port = docker_services.port_for("gitlab", 80)
165123

124+
logging.info("Waiting for GitLab container to come up...")
166125
docker_services.wait_until_responsive(
167-
timeout=200, pause=5, check=lambda: check_is_alive("gitlab-test")
126+
timeout=300, pause=5, check=lambda: check_is_alive("gitlab-test")
168127
)
128+
logging.info("GitLab container is now up")
169129

170130
token = set_token("gitlab-test", fixture_dir=fixture_dir)
171131

@@ -188,12 +148,136 @@ def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_
188148
def gl(gitlab_config):
189149
"""Helper instance to make fixtures and asserts directly via the API."""
190150

151+
logging.info("Create python-gitlab gitlab.Gitlab object")
191152
instance = gitlab.Gitlab.from_config("local", [gitlab_config])
192-
reset_gitlab(instance)
153+
154+
# wait for any running busy sidekiq processes to complete
155+
for count in range(TIMEOUT):
156+
time.sleep(SLEEP_INTERVAL)
157+
busy = False
158+
processes = instance.sidekiq.process_metrics()["processes"]
159+
for process in processes:
160+
if process["busy"]:
161+
logging.info(f"sidekiq: count: {count} process_busy: {process['busy']}")
162+
busy = True
163+
if not busy:
164+
logging.info(f"sidekiq idle check completed after {count} iterations")
165+
break
166+
167+
if is_gitlab_ee(instance):
168+
logging.info("GitLab EE detected")
169+
# NOTE(jlvillal): By default in GitLab EE it will wait 7 days before
170+
# deleting a group. Change it to 0 days.
171+
settings = instance.settings.get()
172+
if settings.deletion_adjourned_period != 0:
173+
settings.deletion_adjourned_period = 0
174+
settings.save()
175+
# Clean up any extraneous resources which may exist
176+
for project in instance.projects.list():
177+
logging.info(f"Mark for deletion project: {project.name!r}")
178+
for deploy_token in project.deploytokens.list():
179+
logging.info(
180+
f"Mark for deletion token: {deploy_token.name!r} in "
181+
f"project: {project.name!r}"
182+
)
183+
deploy_token.delete()
184+
project.delete()
185+
for group in instance.groups.list():
186+
logging.info(f"Mark for deletion group: {group.name!r}")
187+
for deploy_token in group.deploytokens.list():
188+
logging.info(
189+
f"Mark for deletion token: {deploy_token.name!r} in "
190+
f"group: {group.name!r}"
191+
)
192+
deploy_token.delete()
193+
group.delete()
194+
for variable in instance.variables.list():
195+
logging.info(f"Mark for deletion variable: {variable.name!r}")
196+
variable.delete()
197+
for user in instance.users.list():
198+
if user.username != "root":
199+
logging.info(f"Mark for deletion user: {user.username!r}")
200+
user.delete(hard_delete=True)
201+
202+
timeout = TIMEOUT
203+
sleep_interval = 0.5
204+
max_iterations = int(timeout / sleep_interval)
205+
206+
# Ensure everything has been reset
207+
start_time = time.perf_counter()
208+
209+
def wait_for_maximum_list_length(
210+
rest_manager: gitlab.base.RESTManager,
211+
description: str,
212+
max_length: int = 0,
213+
should_delete_func=lambda x: True,
214+
) -> None:
215+
"""Wait for the list() length to be no greater than expected maximum or fail
216+
test if timeout is exceeded"""
217+
for count in range(max_iterations):
218+
items = rest_manager.list()
219+
logging.info(
220+
f"Iteration: {count}: items in {description}: {[x.name for x in items]}"
221+
)
222+
for item in items:
223+
if should_delete_func(item):
224+
logging.info(
225+
f"Marking again for deletion {description}: {item.name!r}"
226+
)
227+
try:
228+
item.delete()
229+
except gitlab.exceptions.GitlabDeleteError as exc:
230+
logging.info(
231+
f"Already marked for deletion: {item.name!r} {exc}"
232+
)
233+
if len(items) <= max_length:
234+
break
235+
time.sleep(sleep_interval)
236+
items = rest_manager.list()
237+
elapsed_time = time.perf_counter() - start_time
238+
if len(items) > max_length:
239+
logging.error(
240+
f"Too many items still remaining and timeout exceeded: {elapsed_time}"
241+
)
242+
assert len(items) <= max_length, (
243+
f"Did not delete required items for {description}. "
244+
f"Elapsed_time: {time.perf_counter() - start_time}\n"
245+
f"items: {[str(x) for x in items]!r}"
246+
)
247+
248+
wait_for_maximum_list_length(rest_manager=instance.projects, description="projects")
249+
wait_for_maximum_list_length(rest_manager=instance.groups, description="groups")
250+
wait_for_maximum_list_length(
251+
rest_manager=instance.variables, description="variables"
252+
)
253+
254+
def should_delete_user(user):
255+
if user.username == "root":
256+
return False
257+
return True
258+
259+
wait_for_maximum_list_length(
260+
rest_manager=instance.users,
261+
description="users",
262+
max_length=1,
263+
should_delete_func=should_delete_user,
264+
)
193265

194266
return instance
195267

196268

269+
def is_gitlab_ee(gl: gitlab.Gitlab) -> bool:
270+
"""Determine if we are running with GitLab EE as opposed to GitLab CE"""
271+
try:
272+
license = gl.get_license()
273+
except gitlab.exceptions.GitlabLicenseError:
274+
license = None
275+
# If we have a license then we assume we are running on GitLab EE
276+
if license:
277+
return True
278+
return False
279+
280+
197281
@pytest.fixture(scope="session")
198282
def gitlab_runner(gl):
199283
container = "gitlab-runner-test"

‎tests/functional/fixtures/.env‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
1-
GITLAB_IMAGE=gitlab/gitlab-ce
2-
GITLAB_TAG=14.5.2-ce.0
1+
GITLAB_IMAGE=gitlab/gitlab-ee
2+
GITLAB_TAG=14.5.2-ee.0
3+
4+
GITLAB_CI_IMAGE=localhost/gitlab-ci
5+
GITLAB_CI_TAG=latest

‎tests/functional/fixtures/Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
ARG GITLAB_IMAGE=get_this_from_env
2+
ARG GITLAB_TAG=get_this_from_env
3+
FROM ${GITLAB_IMAGE}:${GITLAB_TAG}
4+
5+
COPY create_license.rb /
6+
RUN ruby /create_license.rb
7+
8+
CMD ["/assets/wrapper"]

‎tests/functional/fixtures/build-dockerfile.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
set -u
4+
set -e
5+
6+
TOP_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
7+
8+
cd "${TOP_DIR}" || exit 1
9+
10+
source .env
11+
12+
docker build \
13+
-t "${GITLAB_CI_IMAGE}:${GITLAB_CI_TAG}" \
14+
--build-arg GITLAB_IMAGE="${GITLAB_IMAGE}" \
15+
--build-arg GITLAB_TAG="${GITLAB_TAG}" \
16+
--no-cache \
17+
"${TOP_DIR}"

‎tests/functional/fixtures/create_license.rb‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# NOTE: As of 2021-12-26 the GitLab Enterprise Edition License has the following
2+
# section:
3+
# Notwithstanding the foregoing, you may copy and modify the Software for development
4+
# and testing purposes, without requiring a subscription.
5+
#
6+
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/LICENSE
7+
#
8+
# This code is strictly intended for use in the testing framework of python-gitlab
9+
10+
# Code inspired by MIT licensed code at: https://github.com/CONIGUERO/gitlab-license.git
11+
12+
require 'openssl'
13+
require 'gitlab/license'
14+
15+
# Generate a 2048 bit key pair.
16+
license_encryption_key = OpenSSL::PKey::RSA.generate(2048)
17+
18+
# Save the private key
19+
File.open("/.license_encryption_key", "w") { |f| f.write(license_encryption_key.to_pem) }
20+
# Save the public key
21+
public_key = license_encryption_key.public_key
22+
File.open("/.license_encryption_key.pub", "w") { |f| f.write(public_key.to_pem) }
23+
File.open("/opt/gitlab/embedded/service/gitlab-rails/.license_encryption_key.pub", "w") { |f| f.write(public_key.to_pem) }
24+
25+
Gitlab::License.encryption_key = license_encryption_key
26+
27+
# Build a new license.
28+
license = Gitlab::License.new
29+
30+
license.licensee = {
31+
"Name" => "python-gitlab-ci",
32+
"Company" => "python-gitlab-ci",
33+
"Email" => "python-gitlab-ci@example.com",
34+
}
35+
36+
# The date the license starts.
37+
license.starts_at = Date.today
38+
# Want to make sure we get at least 1 day of usage. Do two days after because if CI
39+
# started at 23:59 we could be expired in one minute if we only did one next_day.
40+
license.expires_at = Date.today.next_day.next_day
41+
42+
# Use 'ultimate' plan so that we can test all features in the CI
43+
license.restrictions = {
44+
:plan => "ultimate",
45+
:id => rand(1000..99999999)
46+
}
47+
48+
# Export the license, which encrypts and encodes it.
49+
data = license.export
50+
51+
File.open("/python-gitlab-ci.gitlab-license", 'w') { |file| file.write(data) }

‎tests/functional/fixtures/docker-compose.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ networks:
66

77
services:
88
gitlab:
9-
image: '${GITLAB_IMAGE}:${GITLAB_TAG}'
9+
image: '${GITLAB_CI_IMAGE}:${GITLAB_CI_TAG}'
1010
container_name: 'gitlab-test'
1111
hostname: 'gitlab.test'
1212
privileged: true # Just in case https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/1350
@@ -31,6 +31,7 @@ services:
3131
gitlab_exporter['enable'] = false
3232
grafana['enable'] = false
3333
letsencrypt['enable'] = false
34+
gitlab_rails['initial_license_file'] = "/python-gitlab-ci.gitlab-license"
3435
ports:
3536
- '8080:80'
3637
- '2222:22'

‎tox.ini‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,13 @@ exclude_lines =
9595
[testenv:cli_func_v4]
9696
deps = -r{toxinidir}/requirements-docker.txt
9797
commands =
98+
{toxinidir}/tests/functional/fixtures/build-dockerfile.sh
9899
pytest --script-launch-mode=subprocess --cov --cov-report xml tests/functional/cli {posargs}
99100

100101
[testenv:py_func_v4]
101102
deps = -r{toxinidir}/requirements-docker.txt
102103
commands =
104+
{toxinidir}/tests/functional/fixtures/build-dockerfile.sh
103105
pytest --cov --cov-report xml tests/functional/api {posargs}
104106

105107
[testenv:smoke]

0 commit comments

Comments
 (0)
X Tutup