Coverage for src/gitlabracadabra/mixins/mirrors.py: 81%

1#

2# Copyright (C) 2019-2025 Mathieu Parent <math.parent@gmail.com>

3#

4# This program is free software: you can redistribute it and/or modify

5# it under the terms of the GNU Lesser General Public License as published by

6# the Free Software Foundation, either version 3 of the License, or

7# (at your option) any later version.

8#

9# This program is distributed in the hope that it will be useful,

10# but WITHOUT ANY WARRANTY; without even the implied warranty of

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12# GNU Lesser General Public License for more details.

13#

14# You should have received a copy of the GNU Lesser General Public License

15# along with this program. If not, see <http://www.gnu.org/licenses/>.

16

17 from __future__ import annotations

18

19 import logging

20 from os.path import isdir

21 from typing import TYPE_CHECKING, Any

22 from urllib.parse import quote

23

24 from pygit2 import GIT_FETCH_PRUNE, Commit, GitError, RemoteCallbacks, Repository, init_repository

25

26 from gitlabracadabra.disk_cache import cache_dir

27 from gitlabracadabra.gitlab.connections import GitlabConnections

28 from gitlabracadabra.matchers import Matcher

29 from gitlabracadabra.objects.object import GitLabracadabraObject

30

31 if TYPE_CHECKING: 31 ↛ 32line 31 didn't jump to line 32 because the condition on line 31 was never true

32 from pygit2 import Reference

33

34

35 GITLAB_REMOTE_NAME = "gitlab"

36 MIRROR_PARAM_URL = "url"

37 MIRROR_DIRECTION_PULL = "pull"

38 MIRROR_REMOTE_NAME_PULL = "pull"

39 PUSH_OPTIONS = "push_options"

40

41 logger = logging.getLogger(__name__)

42

43

44 class MirrorsMixin(GitLabracadabraObject):

45 """Object with mirrors."""

46

47 def _process_mirrors(

48 self,

49 param_name: str,

50 param_value: Any,

51 *,

52 dry_run: bool = False,

53 skip_save: bool = False,

54 ) -> None:

55 """Process the mirrors param.

56

57 Args:

58 param_name: "mirrors".

59 param_value: List of mirror dicts.

60 dry_run: Dry run.

61 skip_save: False.

62 """

63 assert param_name == "mirrors" # noqa: S101

64 assert not skip_save # noqa: S101

65

66 pull_mirror_count = 0

67 self._init_repo()

68 self._fetch_remote(

69 GITLAB_REMOTE_NAME,

70 self.connection.pygit2_remote_callbacks,

71 )

72 for mirror in param_value:

73 direction = mirror.get("direction", MIRROR_DIRECTION_PULL)

74 push_options = mirror.get(PUSH_OPTIONS, [])

75 if "skip_ci" in mirror: 75 ↛ 76line 75 didn't jump to line 76 because the condition on line 75 was never true

76 push_options.append("ci.skip")

77 if direction == MIRROR_DIRECTION_PULL: 77 ↛ 88line 77 didn't jump to line 88 because the condition on line 77 was always true

78 if pull_mirror_count > 0: 78 ↛ 79line 78 didn't jump to line 79 because the condition on line 78 was never true

79 logger.warning(

80 "[%s] NOT Pulling mirror: %s (Only first pull mirror is processed)",

81 self._name,

82 mirror[MIRROR_PARAM_URL],

83 )

84 continue

85 self._pull_mirror(mirror, push_options, dry_run=dry_run)

86 pull_mirror_count += 1

87 else:

88 logger.warning(

89 "[%s] NOT Pushing mirror: %s (Not supported yet)",

90 self._name,

91 mirror[MIRROR_PARAM_URL],

92 )

93

94 def _init_repo(self) -> None:

95 """Init the cache repository."""

96 web_url_slug = quote(self.web_url(), safe="")

97 repo_dir = str(cache_dir("") / web_url_slug)

98 if isdir(repo_dir): 98 ↛ 99line 98 didn't jump to line 99 because the condition on line 98 was never true

99 self._repo = Repository(repo_dir)

100 else:

101 logger.debug(

102 "[%s] Creating cache repository in %s",

103 self._name,

104 repo_dir,

105 )

106 self._repo = init_repository(repo_dir, bare=True)

107 try:

108 self._repo.remotes[GITLAB_REMOTE_NAME] # type: ignore[attr-defined]

109 except KeyError:

110 self._repo.remotes.create( # type: ignore[attr-defined]

111 GITLAB_REMOTE_NAME,

112 self.web_url(),

113 "+refs/heads/*:refs/remotes/gitlab/heads/*",

114 )

115 self._repo.remotes.add_fetch( # type: ignore[attr-defined]

116 GITLAB_REMOTE_NAME,

117 "+refs/tags/*:refs/remotes/gitlab/tags/*",

118 )

119 self._repo.remotes.add_push(GITLAB_REMOTE_NAME, "+refs/heads/*:refs/heads/*") # type: ignore[attr-defined]

120 self._repo.remotes.add_push(GITLAB_REMOTE_NAME, "+refs/tags/*:refs/tags/*") # type: ignore[attr-defined]

121 self._repo.config["remote.gitlab.mirror"] = True # type: ignore[attr-defined]

122

123 def _fetch_remote(

124 self,

125 name: str,

126 remote_callbacks: RemoteCallbacks | None = None,

127 ) -> None:

128 """Fetch the repo with the given name.

129

130 Args:

131 name: Remote name.

132 remote_callbacks: Credentials and certificate check as pygit2.RemoteCallbacks.

133 """

134 remote = self._repo.remotes[name] # type: ignore[attr-defined]

135 try:

136 # https://gitlab.com/gitlabracadabra/gitlabracadabra/-/issues/25

137 remote.fetch(

138 refspecs=remote.fetch_refspecs,

139 callbacks=remote_callbacks,

140 prune=GIT_FETCH_PRUNE,

141 proxy=True,

142 )

143 except TypeError:

144 # proxy arg in pygit2 1.6.0

145 logger.warning(

146 "[%s] Ignoring proxy for remote=%s refs=%s: requires pygit2>=1.6.0",

147 self._name,

148 name,

149 ",".join(remote.fetch_refspecs),

150 )

151 remote.fetch(

152 refspecs=remote.fetch_refspecs,

153 callbacks=remote_callbacks,

154 prune=GIT_FETCH_PRUNE,

155 )

156

157 def _pull_mirror(self, mirror: dict, push_options: list[str], *, dry_run: bool) -> None:

158 """Pull from the given mirror and push.

159

160 Args:

161 mirror: Current mirror dict.

162 push_options: push options.

163 dry_run: Dry run.

164 """

165 try:

166 self._repo.remotes[MIRROR_REMOTE_NAME_PULL] # type: ignore[attr-defined]

167 except KeyError:

168 self._repo.remotes.create( # type: ignore[attr-defined]

169 MIRROR_REMOTE_NAME_PULL,

170 mirror[MIRROR_PARAM_URL],

171 "+refs/heads/*:refs/heads/*",

172 )

173 self._repo.remotes.add_fetch( # type: ignore[attr-defined]

174 MIRROR_REMOTE_NAME_PULL,

175 "+refs/tags/*:refs/tags/*",

176 )

177 self._repo.config["remote.pull.mirror"] = True # type: ignore[attr-defined]

178 remote_callbacks = None

179 pull_auth_id = mirror.get("auth_id")

180 if pull_auth_id: 180 ↛ 181line 180 didn't jump to line 181 because the condition on line 180 was never true

181 remote_callbacks = GitlabConnections().get_connection(pull_auth_id).pygit2_remote_callbacks

182 self._fetch_remote(MIRROR_REMOTE_NAME_PULL, remote_callbacks)

183 for ref in self._repo.references.objects: # type: ignore[attr-defined]

184 self._sync_ref(mirror, ref, push_options, dry_run=dry_run)

185

186 def _sync_ref(

187 self,

188 mirror: dict,

189 ref: Reference,

190 push_options: list[str],

191 *,

192 dry_run: bool,

193 ) -> None:

194 """Synchronize the given branch or tag.

195

196 Args:

197 mirror: Current mirror dict.

198 ref: reference objects.

199 push_options: push options.

200 dry_run: Dry run.

201 """

202 if ref.name.startswith("refs/heads/"):

203 ref_type = "head"

204 ref_type_human = "branch"

205 ref_type_human_plural = "branches"

206 elif ref.name.startswith("refs/tags/"): 206 ↛ 211line 206 didn't jump to line 211 because the condition on line 206 was always true

207 ref_type = "tag"

208 ref_type_human = "tag"

209 ref_type_human_plural = "tags"

210 else:

211 return

212 shorthand = ref.name.split("/", 2)[2]

213

214 # Ref mapping

215 dest_shortand: str | None = shorthand

216 if ref_type_human_plural in mirror:

217 dest_shortand = None

218 mappings: list[dict[str, str | list[str]]] = mirror.get(ref_type_human_plural) # type: ignore

219 for mapping in mappings:

220 matcher = Matcher(

221 mapping.get("from", ""),

222 None,

223 log_prefix=f"[{self._name}] {mirror[MIRROR_PARAM_URL]} {ref_type_human_plural}",

224 )

225 matches = matcher.match([shorthand])

226 if matches:

227 to_param = mapping.get("to", shorthand)

228 dest_shortand = matches[0].expand(to_param)

229 push_options = mapping.get(PUSH_OPTIONS, push_options) # type: ignore

230 break

231

232 if dest_shortand is None:

233 return

234

235 pull_commit = ref.peel(Commit).id

236 gitlab_ref = self._repo.references.get( # type: ignore[attr-defined]

237 f"refs/remotes/gitlab/{ref_type}s/{dest_shortand}",

238 )

239 try:

240 gitlab_commit = gitlab_ref.peel(Commit).id

241 except AttributeError:

242 gitlab_commit = None

243 if pull_commit == gitlab_commit: 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true

244 return

245 if dry_run: 245 ↛ 246line 245 didn't jump to line 246 because the condition on line 245 was never true

246 logger.info(

247 "[%s] %s NOT Pushing %s %s to %s: %s -> %s (dry-run)",

248 self._name,

249 mirror[MIRROR_PARAM_URL],

250 ref_type_human,

251 shorthand,

252 dest_shortand,

253 gitlab_commit,

254 str(pull_commit),

255 )

256 return

257 logger.info(

258 "[%s] %s Pushing %s %s to %s: %s -> %s",

259 self._name,

260 mirror[MIRROR_PARAM_URL],

261 ref_type_human,

262 shorthand,

263 dest_shortand,

264 gitlab_commit,

265 str(pull_commit),

266 )

267 refspec = f"{ref.name}:refs/{ref_type}s/{dest_shortand}"

268 try:

269 self._push_remote(

270 GITLAB_REMOTE_NAME,

271 [refspec],

272 push_options,

273 self.connection.pygit2_remote_callbacks,

274 )

275 except GitError as err:

276 logger.error(

277 "[%s] Unable to push remote=%s refs=%s: %s",

278 self._name,

279 GITLAB_REMOTE_NAME,

280 refspec,

281 err,

282 )

283

284 def _push_remote(

285 self,

286 name: str,

287 refs: list[str],

288 push_options: list[str],

289 remote_callbacks: RemoteCallbacks | None,

290 ) -> None:

291 """Push to the repo with the given name.

292

293 Args:

294 name: Remote name.

295 refs: refs list.

296 push_options: push options.

297 remote_callbacks: Credentials and certificate check as pygit2.RemoteCallbacks.

298 """

299 remote = self._repo.remotes[name] # type: ignore[attr-defined]

300 kwargs = {

301 "specs": refs,

302 "callbacks": remote_callbacks,

303 "proxy": True,

304 }

305 if push_options:

306 kwargs[PUSH_OPTIONS] = push_options

307 try:

308 remote.push(**kwargs)

309 except TypeError:

310 # push_options arg in pygit2 1.16.0

311 logger.warning(

312 "[%s] Ignoring push options %s for remote=%s refs=%s: requires pygit2>=1.16.0",

313 self._name,

314 ",".join(push_options),

315 name,

316 ",".join(refs),

317 )

318 kwargs.pop(PUSH_OPTIONS)

319 else:

320 return

321 try:

322 remote.push(**kwargs)

323 except TypeError:

324 # proxy arg in pygit2 1.6.0

325 logger.warning(

326 "[%s] Ignoring proxy for remote=%s refs=%s: requires pygit2>=1.6.0",

327 self._name,

328 name,

329 ",".join(refs),

330 )

331 kwargs.pop("proxy")

332 remote.push(**kwargs)