From b07ac1f302b14ab9396df28852a0fc07fd800e69e0329c65fa746bb7801f84fd Mon Sep 17 00:00:00 2001 From: Andrii Nikitin Date: Fri, 13 Dec 2024 11:33:23 +0000 Subject: [PATCH] - Update to version 1.089: * Fix presense in header x-media-version when empty (#548) * Add /app/efficiency reporting cache hit rate (#547) * Do not verify redirects of unversioned files on HEAD requests (#546) OBS-URL: https://build.opensuse.org/package/show/openSUSE:infrastructure:MirrorCache/MirrorCache?expand=0&rev=122 --- .gitattributes | 23 + .gitignore | 1 + MirrorCache-1.077.obscpio | 3 + MirrorCache-1.079.obscpio | 3 + MirrorCache-1.081.obscpio | 3 + MirrorCache-1.084.obscpio | 3 + MirrorCache-1.088.obscpio | 3 + MirrorCache-1.089.obscpio | 3 + MirrorCache-tmpfilesd.conf | 4 + MirrorCache-user.conf | 3 + MirrorCache.changes | 882 +++++++++++++++++++++++++++++++++++++ MirrorCache.obsinfo | 4 + MirrorCache.spec | 103 +++++ _service | 20 + cache.tar.xz | 3 + update-cache.sh | 24 + 16 files changed, 1085 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 MirrorCache-1.077.obscpio create mode 100644 MirrorCache-1.079.obscpio create mode 100644 MirrorCache-1.081.obscpio create mode 100644 MirrorCache-1.084.obscpio create mode 100644 MirrorCache-1.088.obscpio create mode 100644 MirrorCache-1.089.obscpio create mode 100644 MirrorCache-tmpfilesd.conf create mode 100644 MirrorCache-user.conf create mode 100644 MirrorCache.changes create mode 100644 MirrorCache.obsinfo create mode 100644 MirrorCache.spec create mode 100644 _service create mode 100644 cache.tar.xz create mode 100644 update-cache.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9b03811 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,23 @@ +## Default LFS +*.7z filter=lfs diff=lfs merge=lfs -text +*.bsp filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.gem filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.jar filter=lfs diff=lfs merge=lfs -text +*.lz filter=lfs diff=lfs merge=lfs -text +*.lzma filter=lfs diff=lfs merge=lfs -text +*.obscpio filter=lfs diff=lfs merge=lfs -text +*.oxt filter=lfs diff=lfs merge=lfs -text +*.pdf filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.rpm filter=lfs diff=lfs merge=lfs -text +*.tbz filter=lfs diff=lfs merge=lfs -text +*.tbz2 filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.ttf filter=lfs diff=lfs merge=lfs -text +*.txz filter=lfs diff=lfs merge=lfs -text +*.whl filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..57affb6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.osc diff --git a/MirrorCache-1.077.obscpio b/MirrorCache-1.077.obscpio new file mode 100644 index 0000000..7fd8fc2 --- /dev/null +++ b/MirrorCache-1.077.obscpio @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d456dc5eec645a4322b2d1cce01fb4f41795e3afe36bdc74b94ecc71cc30381b +size 1213452 diff --git a/MirrorCache-1.079.obscpio b/MirrorCache-1.079.obscpio new file mode 100644 index 0000000..2f57052 --- /dev/null +++ b/MirrorCache-1.079.obscpio @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6879daa0f4ae5732bd00b2bbb34e57aa249075c0e3c455e9057342583c64aeb6 +size 1223692 diff --git a/MirrorCache-1.081.obscpio b/MirrorCache-1.081.obscpio new file mode 100644 index 0000000..8f07b58 --- /dev/null +++ b/MirrorCache-1.081.obscpio @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5942c37a495ecfff56046d10ba71330b755ea6e4dca2a92addd69e40853a9af +size 1225740 diff --git a/MirrorCache-1.084.obscpio b/MirrorCache-1.084.obscpio new file mode 100644 index 0000000..4fa9d80 --- /dev/null +++ b/MirrorCache-1.084.obscpio @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae5f322e20b7248b5511475cfb404c47969c3c4466503bbd51a9736d84d7c59 +size 1230348 diff --git a/MirrorCache-1.088.obscpio b/MirrorCache-1.088.obscpio new file mode 100644 index 0000000..2fbace9 --- /dev/null +++ b/MirrorCache-1.088.obscpio @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08841b0a4a028f95f2f54be1b24738dcd54fcdd3738bacad12b2e1e3246af23f +size 1287180 diff --git a/MirrorCache-1.089.obscpio b/MirrorCache-1.089.obscpio new file mode 100644 index 0000000..a14cb88 --- /dev/null +++ b/MirrorCache-1.089.obscpio @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:674d11a5ee6175f43b1934cc4d5f197d100a32e61804f85a5eae19da20434b9b +size 1299980 diff --git a/MirrorCache-tmpfilesd.conf b/MirrorCache-tmpfilesd.conf new file mode 100644 index 0000000..71251ee --- /dev/null +++ b/MirrorCache-tmpfilesd.conf @@ -0,0 +1,4 @@ +# Type Path Mode UID GID Age Argument +d /var/lib/mirrorcache 0750 mirrorcache mirrorcache - - +d /var/lib/mirrorcache/tmp 0750 mirrorcache mirrorcache 2h +d /run/mirrorcache 0750 mirrorcache mirrorcache - - diff --git a/MirrorCache-user.conf b/MirrorCache-user.conf new file mode 100644 index 0000000..f54acb8 --- /dev/null +++ b/MirrorCache-user.conf @@ -0,0 +1,3 @@ +# Type Name ID GECOS [HOME] +u mirrorcache - "MirrorCache" /var/lib/mirrorcache +g mirrorcache diff --git a/MirrorCache.changes b/MirrorCache.changes new file mode 100644 index 0000000..a44dcd2 --- /dev/null +++ b/MirrorCache.changes @@ -0,0 +1,882 @@ +------------------------------------------------------------------- +Fri Dec 13 11:29:33 UTC 2024 - Andrii Nikitin + +- Update to version 1.089: + * Fix presense in header x-media-version when empty (#548) + * Add /app/efficiency reporting cache hit rate (#547) + * Do not verify redirects of unversioned files on HEAD requests (#546) + +------------------------------------------------------------------- +Mon Dec 2 18:37:12 UTC 2024 - Elisei Roca + +- Update assets cache and improve update-cache.sh script + +------------------------------------------------------------------- +Thu Nov 28 14:27:21 UTC 2024 - Andrii Nikitin + +- Update to version 1.088: + * Add parameter strict to /app/package + * Use urandom for random_string + * backstage: properly close DB connections in minion jobs + * systemd: for background services LOG_LEVEL=error and MemoryMax + * ci: Bump mariadb experimental version to 11.7 + +------------------------------------------------------------------- +Sat Nov 23 12:10:19 UTC 2024 - Andrii Nikitin + +- Update to version 1.087: + * Fix enormous query retrieving existing packages (#536) + +------------------------------------------------------------------- +Fri Nov 22 09:46:10 UTC 2024 - Andrii Nikitin + +- Update to version 1.086: + * Extra filters to /app/package (#535) + +------------------------------------------------------------------- +Thu Nov 21 10:36:04 UTC 2024 - Andrii Nikitin + +- Update to version 1.085: + * Add /app/package (#534) + * Make sure services are started in setup.md + +------------------------------------------------------------------- +Wed Nov 06 08:45:02 UTC 2024 - Andrii Nikitin + +- Update to version 1.084: + * systemd: MOJO_TMPDIR=/var/lib/mirrorcache/tmp + * Smart handle of local symlinks + +------------------------------------------------------------------- +Fri Oct 25 15:02:11 UTC 2024 - Andrii Nikitin + +- Update to version 1.083: + * Escape regex in browse.html.ep (#527) + +------------------------------------------------------------------- +Thu Oct 24 10:25:03 UTC 2024 - Andrii Nikitin + +- Update to version 1.082: + * Redirect all local symlinks + * Detect only symlinks with specific name during remote scan + * fix encoding in the mirrors report when using MariaDB + * systemd: configure MOJO_TMP_DIR=/tmp/mirrorcache + * Set permission 750 to /etc/mirrorcache + +------------------------------------------------------------------- +Fri Aug 09 09:48:52 UTC 2024 - Andrii Nikitin + +- Update to version 1.081: + * Add /rest/project_disk_usage route (#510) + * fix redirect of unversioned files when accept has zsync (#509) + +------------------------------------------------------------------- +Thu Aug 01 08:59:31 UTC 2024 - Andrii Nikitin + +- Update to version 1.080: + * Add /rest/project_last_modified route (#506) + * Display project size in human friendly format (#503) + * Add flag mysql_enable_utf8mb4 when connectin to MariaDB (#505) + +------------------------------------------------------------------- +Thu Jul 25 07:40:38 UTC 2024 - Andrii Nikitin + +- Update to version 1.079: + * Show size of projects (#502) + +------------------------------------------------------------------- +Thu Jul 18 08:23:40 UTC 2024 - Andrii Nikitin + +- Update to version 1.078: + * Respect If-Modified-Since header (#501) + * Add content-disposition header in torrent responses (#497) + * Reuse file extention instead of url parameter in redirects (#497) + * Allow caches to serve files while re-checking (#492) + +------------------------------------------------------------------- +Thu Jun 20 05:47:14 UTC 2024 - Andrii Nikitin + +- Update to version 1.077: + * UI fix db sync timezone (#487) + * tmpfiles: Let working and run directories be always created (#488) + * Switch CI to 15.6 (#489) + * Fix doubleslash in mirrors report (#490) + +------------------------------------------------------------------- +Tue May 21 14:18:43 UTC 2024 - Andrii Nikitin + +- Update to version 1.076: + * Fix duplicate values in x-media-version response header + * ci: Temporarily disable systemd tests + +------------------------------------------------------------------- +Tue May 21 11:00:26 UTC 2024 - Andrii Nikitin + +- Update to version 1.075: + * Add url parameters for report/download + * ui: User friendly status of DB sync for a directory + * Add Etag and X-MEDIA-VERSION for folders + * Add response header X-MEDIA-VERSION + * Add Etag when rendering files + * Prioritize scanning of projects + * Make VPN_PREFIX a list + * Add WorkingDir to the subtree.service + * Add config for mojo tmpfiles + * fix redirect for mirrorlist parameter for zsync files + +------------------------------------------------------------------- +Thu Apr 04 10:32:53 UTC 2024 - Andrii Nikitin + +- Update to version 1.074: + * Rework mirrors report generation + * fix files mtime descrepancy when syncing over http + * Fix metalink for small files + * fix serving linked folders when dest changes + +------------------------------------------------------------------- +Thu Mar 07 15:21:08 UTC 2024 - Andrii Nikitin + +- Update to version 1.073: + * Track mirror propagation using time of job + * Add project propagation details /app/server_rollout/:version + * Do not show projects with negative priority + * Show only relevant mirrors in /report/mirrors/:proj + * fix .mirrorlist doesnt always work for repomd.xml + * ui: add prio column to /app/project + +------------------------------------------------------------------- +Thu Feb 29 11:32:53 UTC 2024 - Andrii Nikitin + +- Update to version 1.072: + * ui: add /app/project/:id with propagation + * Rework rollout to include subfolders in repo projects + +------------------------------------------------------------------- +Thu Feb 08 10:05:38 UTC 2024 - Andrii Nikitin + +- Update to version 1.071: + * Add rollout tracking for iso and repo projects (#451) + * Allow redirects in MirrorFileCheck task (#450) + * Add timeout for hashes import job (#449) + * Adjust regex for folder mirrorlist (#449) + +------------------------------------------------------------------- +Thu Dec 07 09:00:51 UTC 2023 - Andrii Nikitin + +- Update to version 1.070: + * ui: Show health on mirror card (#439) + * fix capabilities sometimes not shown in /report/mirrors (#438) + +------------------------------------------------------------------- +Thu Nov 30 09:45:16 UTC 2023 - Andrii Nikitin + +- Update to version 1.069: + * t: Adjust location of environ/mc/db links (#437) + * t: Make 09-stability-02 more robust (#437) + * t: Bump MariaDB experimental version (#437) + * t: Switch environ to package from github repo (#437) + * Add generation time to /report/mirrors (#436) + * Remove postgres replication support (#435) + * fix: mirrors with capability declaraion might disappear in /rest/server (#434) + +------------------------------------------------------------------- +Thu Nov 16 16:56:06 UTC 2023 - Andrii Nikitin - 1.068 + +- Update to version 1.068: + * Add diagnostic job for checking presence of a file on mirrors + * Adjust switch Digest::Zsync to separate package + * Update regexps to handle zchunk, zstd and sha512 + * Improve link detection in the same folder + * Add a Vary Fastly-SSL + +------------------------------------------------------------------- +Thu Oct 26 07:56:51 UTC 2023 - Andrii Nikitin + +- Update to version 1.067: + * fix handling combinations of metalink / zsync / mirrorlist / symlinks (#421) + +------------------------------------------------------------------- +Thu Oct 19 11:37:00 UTC 2023 - Andrii Nikitin + +- Update to version 1.066: + * Centalized mirror management (#417) + * Fix surepfluous .zsync in symlink redirect (#419) + * t: Move container tests to from leap 15.4 to 15.5 + +------------------------------------------------------------------- +Thu Sep 28 10:12:08 UTC 2023 - Andrii Nikitin + +- Update to version 1.065: + * ui: Use https when applicable on mirrors report (#411) + * ui: Show country flag in mirror report (#411) + * ui: Tweak mirrors report (#411) + * ui: Remove tidy warnings from openSUSE branding (#411) + * ui: Add mirror details page (#411) + * Remove dark theme css code for Mirrors page (#408) + +------------------------------------------------------------------- +Thu Sep 14 06:49:28 UTC 2023 - Andrii Nikitin + +- Update to version 1.064: + * Remove must-revalidate from cache-control (#407) + * Add handling x-metalink-limit request header (#407) + * Add handling x-country-code request header (#407) + * Fix escaping in regexps (#406) + +------------------------------------------------------------------- +Thu Aug 31 07:36:54 UTC 2023 - Andrii Nikitin + +- Update to version 1.063: + * Add cache-control options (#404) + +------------------------------------------------------------------- +Thu Aug 03 08:42:06 UTC 2023 - Andrii Nikitin + +- Update to version 1.062: + * Add GEOIP config (#399) + * Add url parameter LIMIT fir number of mirrors in metalink (#399) + * Improve download report for multigroups (#398) + +------------------------------------------------------------------- +Thu Jul 20 07:08:51 UTC 2023 - Andrii Nikitin + +- Update to version 1.061: + * Add support of some parameters in ini file (#397) + * Fix html mirror report (#394) + +------------------------------------------------------------------- +Thu Jun 15 06:46:53 UTC 2023 - Andrii Nikitin + +- Update to version 1.060: + * Tolerate DB downtime (#392) + * Cache mirror report in case if DB or subsidiary is down (#393) + * Fix value of METALINK_GREEDY in meta4 (#389) + * Honor links for MIRRORCACHE_REDIRECT (#390) + * Rework and fix /rest/stat (#387) + +------------------------------------------------------------------- +Mon May 08 13:47:15 UTC 2023 - Andrii Nikitin + +- Update to version 1.059: + * Consider links when redirecting to the root (#374) + * Add backend for mirror-counries relation (#375) + * Use optinally default conf file in subtree service (#379) + * rework salt test to use formula (#378) + * ci: Switch systemd tests to github actions (#379) + * Switch to podman from docker in systemd tests (#379) + +------------------------------------------------------------------- +Thu Apr 13 10:36:14 UTC 2023 - Andrii Nikitin + +- Update to version 1.058: + * Add Vary header into http responses (#373) + * Add experimental feature mirrorlist for folder (#372) + * Fix redirect on remote MirrorCache (#372) + +------------------------------------------------------------------- +Thu Apr 06 06:31:14 UTC 2023 - Andrii Nikitin + +- Update to version 1.057: + * sql: Optimize lookup for server_project (#369) + * Avoid deadlock in minion pubsub code (#371) + * Avoid deadlock in cleanup job deleting from folder_diff (#371) + * ci: Switch ci container from 15.3 to 15.4 (#370) + +------------------------------------------------------------------- +Thu Feb 09 09:41:42 UTC 2023 - Andrii Nikitin + +- Update to version 1.056: + * Improve symlink handling when remote root references another remote root (#353) + * ui: add content-disposition inline to render_file (#355) + * Don't send users to Bugzilla by @AdamMajer (#357) + * Add leap 15.5 to project list (#358) + * Add GNOME iso to Current pattern (#356) + +------------------------------------------------------------------- +Thu Feb 02 12:01:48 UTC 2023 - Andrii Nikitin + +- Update to version 1.055: + * Handle redirects with RootRemote (#351) + * Add sha512 to hash (#352) + * Do not urlencode plus character (#352) + +------------------------------------------------------------------- +Fri Jan 27 13:16:17 UTC 2023 - Andrii Nikitin + +- Update to version 1.054: + * Optimise sql for erasing folder_diff_server (#347) + * Fix huge files rendered from REDIRECT_HUGE only for metalink (#348) + +------------------------------------------------------------------- +Thu Jan 26 07:23:04 UTC 2023 - Andrii Nikitin + +- Update to version 1.053: + * Add MIRROR_PROVIDER to share mirror list (#343) + * Fix REDIRECT_HUGE with local files (#345) + * ui: Dont encode colon character in file names (#346) + +------------------------------------------------------------------- +Thu Jan 12 08:43:17 UTC 2023 - Andrii Nikitin + +- Update to version 1.052: + * Detect redirects with remote root (#336) + * Add json parsing to FolderSync task (#337) + * Redirect Current.iso when no nfs mount (#339) + * Improve sorting of files in JavaScript (#340) + * Redirect huge files (#342) + * Secure last_id for jobs monitoring stat (#341) + +------------------------------------------------------------------- +Thu Dec 29 09:21:35 UTC 2022 - Andrii Nikitin + +- Update to version 1.051: + * Add P/GLOB and REGEX parameters to file listing (#334) + * ui: Move mirrorlist url to size/mtime column (#335) + * Improve url escaping in ui and redirects (#335) + +------------------------------------------------------------------- +Thu Dec 22 10:15:27 UTC 2022 - Andrii Nikitin + +- Update to version 1.050: + * Show description of top level folders for openSUSE (#333) + * Do not access DB in local render for / (#333) + * Improve layout for small screen size (#333) + * Implement realpath for remote root with nfs (#331) + +------------------------------------------------------------------- +Thu Dec 15 12:26:30 UTC 2022 - Andrii Nikitin + +- Update to version 1.049: + * Unify display of current path in /browse /download and .mirrorlist (#327) + * Show only hostname in mirrorlist url (#329) + * Set content-type for zsync (#328) + * Discover folders using realpath (#330) + +------------------------------------------------------------------- +Thu Nov 24 13:06:57 UTC 2022 - Andrii Nikitin + +- Update to version 1.048: + * Fix sorting by size in /report/download (#324) + * Change page size in /report/download (#324) + * Improve sorting in /browse (#324) + * Fix map icon not shown properly in .mirrorlist (#324) + +------------------------------------------------------------------- +Thu Nov 24 10:17:35 UTC 2022 - Andrii Nikitin + +- Update to version 1.047: + * Add /browse route to navigate folders with ajax (#322) + * Render TOP_FOLDER with /browse if browser (#322) + * Add ajax table for the one dimension download reports (#318) + * Cleanup stat older than STAT_KEEP_DAYS (8) (#319) + * Fix db password from ini file wasnt taken in all places (#323) + * Add option for enabling Mojo Status Plugin (#318) + +------------------------------------------------------------------- +Thu Nov 17 06:21:34 UTC 2022 - Andrii Nikitin + +- Update to version 1.046: + * Correctly render mirrorlist in browser when file is unknown + * Allow rendering .zsync files + * Render selected mirrors with osm in .mirrorlist + +------------------------------------------------------------------- +Fri Nov 11 10:42:10 UTC 2022 - Andrii Nikitin +- Add temporary patch for default database name + * 0001-Default-db-name.patch + +------------------------------------------------------------------- +Fri Nov 11 09:42:10 UTC 2022 - Andrii Nikitin + +- Update to version 1.045: + * Add basic support for config from ini file (#314) + * Drop support for rsync in MIRRORCACHE_ROOT (#314) + * Allow html tags in CUSTOM_FOOTER_MESSAGE (#314) + * Fix os version in report generation on mariadb (#314) + * Fix zsync file generation missing empty line after header (#314) + * Fix download report including stats only from begin of hour (#315) + +------------------------------------------------------------------- +Thu Oct 27 13:16:53 UTC 2022 - Andrii Nikitin + +- Update to version 1.044: + * Move minion stats to unauthenticated route (#313) + * Fix occasional duplicate error in agg_download (#313) + +------------------------------------------------------------------- +Thu Oct 27 09:32:56 UTC 2022 - Andrii Nikitin + +- Update to version 1.043: + * Add backend for download size reports (#309) + * Add mirrors to zsync response (#312) + * Prefer zsync over metalink responses (#312) + * Avoid race condition when new files are discovered while mirrors are scanned (#311) + * Fix generation of zsync hashes for big files (#310) + * t: add manual test for local folder (#308) + * Retry hashes import if a new file hasnt got any (#307) + * Add trailing slash for top folders (#306) - bmwiedermann + +------------------------------------------------------------------- +Thu Sep 29 09:47:07 UTC 2022 - Andrii Nikitin + +- Update to version 1.042: + * Add support for metalink4 format (#303) + * Add url parameter IP to simulate geo experience (#304) + * t: Fix CI after changes in async rendering in Mojo (#303) + +------------------------------------------------------------------- +Tue Sep 27 11:42:51 UTC 2022 - Andrii Nikitin + +- Update to version 1.041: + * Fix metalink missing 'verification' tag (#302) + +------------------------------------------------------------------- +Thu Sep 22 14:41:52 UTC 2022 - Andrii Nikitin + +- Update to version 1.040: + * Add parameter METALINK_GREEDY (#300) + * Optimization: assume package managers never want to render dir (#300) + * Fix mirror picked despite disabled in server_project table (#301) + * Avoid by and ru mirrors for ua requests (#298) + +- Remove temporary patch, it is now included in sources + * 0001-Avoid-by-and-ru-mrrors-for-ua-requests.patch + +------------------------------------------------------------------- +Thu Sep 1 14:29:10 UTC 2022 - Elisei Roca + +- Update assets cache + +------------------------------------------------------------------- +Thu Sep 1 14:09:37 UTC 2022 - Elisei Roca + +- Add temporary patch to avoid by and ru mirrors for ua requests + * 0001-Avoid-by-and-ru-mrrors-for-ua-requests.patch + +------------------------------------------------------------------- +Fri Aug 26 13:17:22 UTC 2022 - Andrii Nikitin + +- Update to version 1.039: + * Fix editing columns with underscore in /app (#297) + * Fix mirror report links in header (#297) + +------------------------------------------------------------------- +Wed Aug 24 08:12:10 UTC 2022 - Andrii Nikitin + +- Update to version 1.038: + * Control order of projects in mirrors report (#294) + * Include subsidiary url into region in /report/mirrors (#295) + * Add mirror report links to header in openSUSE template (#296) + * Add operator to mirrors report (#296) + * Add column sponsor to server table (#296) + +------------------------------------------------------------------- +Thu Aug 11 15:10:05 UTC 2022 - Andrii Nikitin + +- Update to version 1.037: + * Add configuration for redirect for projects (#293) + +------------------------------------------------------------------- +Fri Aug 05 12:14:42 UTC 2022 - Andrii Nikitin + +- Update to version 1.036: + * Fix mirrors report when missing folders (#288) + * Tweak systemd files (#291) + * Rework mirrors report (#290) + * Rescan if no country hit (#292) + +------------------------------------------------------------------- +Thu Jun 23 04:55:11 UTC 2022 - Andrii Nikitin + +- Update to version 1.035: + * Add project checkboxes to mirror report (#280) + * Collect report from subsidiaries (#282) + * Fix incorrect path in metalink for origin file (#281) + * Add config CUSTOM_FOOTER_MESSAGE (#283) + * CircleCI: ignore gh-pages branch (#279) + +------------------------------------------------------------------- +Tue Jun 07 09:05:15 UTC 2022 - Andrii Nikitin + +- Update to version 1.034: + * Provide torrent (#278) + * Add report for projects on mirrors (#276) + * Fix render when metalink is optional (#277) + +------------------------------------------------------------------- +Thu May 19 07:14:05 UTC 2022 - Andrii Nikitin + +- Update to version 1.033: + * Add various patterns for files for which mirrors shouldnt be used (#275) + * Fix regexp in latest_misses sql so *-primary.xml wasnt requesting scans (#275) + * Add MyMirrors menu to openSUSE branding (#275) + * Introduce MIRRORCACHE_VPN variable which has higher priority thant MIRRORCACHE_VPN_PREFIX (#275) + * Allow regular users add and manage own mirrors (#274) + * Add parameter MIRRORCACHE_MAX_PATH (#274) + +------------------------------------------------------------------- +Thu May 12 07:31:11 UTC 2022 - Andrii Nikitin + +- Update to version 1.032: + * Set MIME type when rendering files (#273) + * Optimize main sql query so it doesnt create tmp disk tables in mariadb (#272) + * Force sql plan in job mirror_check_from_stat for mariadb (#272) + * Add shared guard to schedule jobs to avoid deadlocks in DB (#272) + +------------------------------------------------------------------- +Thu May 05 01:57:15 UTC 2022 - Andrii Nikitin + +- Update to version 1.031: + * Do not search mirrors for small files with local root (#271) + * Improve detection of https behind proxy (#271) + +------------------------------------------------------------------- +Thu Apr 28 04:52:32 UTC 2022 - Andrii Nikitin + +- Update to version 1.030: + * Local publisher origin in metalink (#267) + * Ignore work requested by rclone (#268) + * Use random mirror for unknown countries (#269) + * Collect statistics for bot activity (#270) + +------------------------------------------------------------------- +Thu Apr 21 01:42:59 UTC 2022 - Andrii Nikitin + +- Update to version 1.029: + * Add support for MariaDB (#265) + * Fix occasional SQL error when mirrorlist for unknown country (#264) + * Fix scheduling folder scan if were not accessed for more than month (#263) + +------------------------------------------------------------------- +Wed Mar 30 15:00:05 UTC 2022 - Andrii Nikitin + +- Update to version 1.028: + * Introduce in-application load balancing read queries to DB replica (#262) + * Fix bigint DB error when inserting hash for big files (#262) + * Move initialization code from before_server_start to execute earlier (#262) + * t: Fix unstable test 03-headquarter-subsidiaries-weight1 by correcting expected values (#262) + * spec: Fix renamed README in the spec file (#262) + +------------------------------------------------------------------- +Wed Mar 16 16:40:33 UTC 2022 - Andrii Nikitin + +- Update to version 1.027: + * Fix auto-disabled mirrors may be still used for redirection (#261) + * Increase logging level for timeouts in PEDANTIC mode (#261) + * Serve CentOS repodata from root only (#261) + +------------------------------------------------------------------- +Wed Mar 09 12:42:33 UTC 2022 - Andrii Nikitin + +- Update to version 1.026: + * Fix message for unknown folder render timeout (#260) + * Support single-country subsidiaries (#259) + * Add table server_stability (#258) + +------------------------------------------------------------------- +Thu Feb 10 11:27:57 UTC 2022 - Andrii Nikitin + +- Update to version 1.025: + * Allow headquarter be in HA subsidiary handling (#255) + * Stat log after render (#256) + * Add SMALL_FILE_SIZE to serve small files directly from ROOT_NFS (#254) + +------------------------------------------------------------------- +Thu Feb 03 09:23:28 UTC 2022 - Andrii Nikitin + +- Update to version 1.024: + * Allow multiple subsidiaries per region (#250) + * Add internal timeouts for mirror checks and rendering (#249, #252) + +------------------------------------------------------------------- +Thu Jan 20 07:46:09 UTC 2022 - Andrii Nikitin + +- Update to version 1.023: + * Add execution_time to stat table (#247) + * Publish docs with mkdocs and github actions (#243) + * Local render when ROOT_NFS (#246) + * Add support for zsync hashes (#245) + * Do not use app->static for rendering (#244) + +------------------------------------------------------------------- +Thu Dec 16 07:13:04 UTC 2021 - Andrii Nikitin + +- Update to version 1.022: + * Add experimental route geoip to rewrite baseurl in .repo (#242) + * Set MirrorCache useragent for scan jobs (#240) + * Add RestartSec to backstage systemd files (#241) + +------------------------------------------------------------------- +Sun Dec 12 13:09:46 UTC 2021 - Andrii Nikitin + +- Update to version 1.021: + * Handle timeouts in mirror_probe_projects (#238) + * Track projects on mirrors and do skip scans accordingly (#237) + +------------------------------------------------------------------- +Thu Dec 09 06:22:21 UTC 2021 - Andrii Nikitin + +- Update to version 1.020: + * Do not try to find mirror outside region when ROOT_COUNTRY is set (#236) + * Special handling unversioned media symlinks (#235) + * Import hash create folder (#233) + +------------------------------------------------------------------- +Thu Dec 02 05:33:04 UTC 2021 - Andrii Nikitin + +- Update to version 1.019: + * Optimize sql for selecting mirror misses (#232) + * Import hashes (#231) + * Fix mirrorlist for subsidiary when in the same db with headquarter (#230) + +------------------------------------------------------------------- +Thu Nov 25 08:56:58 UTC 2021 - Andrii Nikitin + +- Update to version 1.018: + * Fix occasional 404 on subtree mirrorlist + * Fix not selecting mirrors for unknown country + * Let Task MirrorProbe work without GeoIP database + +------------------------------------------------------------------- +Thu Nov 11 09:49:34 UTC 2021 - Andrii Nikitin + +- Update to version 1.017: + * Find mirror from rest of world + * Add manual test for subsidiaries + +------------------------------------------------------------------- +Thu Nov 04 08:21:40 UTC 2021 - Andrii Nikitin + +- Update to version 1.016: + * Add script for manual testing download.o.o + * Add route /rest/scan_tree + * Add job for recursive scan (#222) + * Introduce variable MIRRORCACHE_TRUST_ADDR (#221) + * Add BACKSTAGE_WORKERS parameter to scripts (#220) + * Fix automatic mirror capability management when two capabilities are disabled for the same mirror (#219) + * Dont try to find mirror for unversioned Debian files (#218) + * Add hypnotoad service (#217) + * Log statistics the old way + * Add stress tests + +------------------------------------------------------------------- +Thu Oct 28 08:56:27 UTC 2021 - Andrii Nikitin + +- Update to version 1.015: + * Add hypnotoad service (#217) + * Log statistics the old way + * Add stress tests + * Add indices on sync_requested and scan_requested to avoid deadlock + * Allow hash.mtime differ for up to 1 min + * Rework estimation of inactive jobs count + +------------------------------------------------------------------- +Thu Oct 21 06:57:03 UTC 2021 - Andrii Nikitin + +- Update to version 1.014: + * Stop rescanning folders which are not requested anymore + * Schedule rescan depending on age of file and last scan (#212) + +------------------------------------------------------------------- +Thu Oct 14 10:14:44 UTC 2021 - Andrii Nikitin + +- Update to version 1.012: + * Rework mirror scan (#211) + * Fix root redirect in subtree mirrorlist + * Show only folders from TOP_FOLDERS variable if defined + * Fix detecting vpn when ipv6 is enabled + +------------------------------------------------------------------- +Thu Oct 07 07:01:48 UTC 2021 - Andrii Nikitin + +- Update to version 1.011: + * Improve openSUSE theme + * Fix sql error when mirror has no files + * Introduce hasall mirror capability + * Change behavior filtering mirrors based on supported IP version + * Fix mirrorlist for the same region + * Schedule scan even when Geo for IP is not detected + * Improve detection of ipv4 + * Fix syntax error after perl upgrade + * Properly redirect for symlinks in subtree + +------------------------------------------------------------------- +Thu Sep 30 07:45:10 UTC 2021 - Andrii Nikitin + +- Update to version 1.010: + * Add openSUSE design theme + * Fix mirror_error event logging + * Fix subtree to work with symlinks + * Package mirrorcache-subtree.service + +------------------------------------------------------------------- +Thu Sep 23 07:30:36 UTC 2021 - Elisei Roca + +- Update to version 1.009: + * Add proper handling of local symlinks + * Erase info about folder on a mirror if it cannot be accessed + +------------------------------------------------------------------- +Thu Sep 16 08:20:18 UTC 2021 - Andrii Nikitin + +- Update to version 1.008: + * Add GeoIP support for IP2Location db format + * Scan all mirrors when mirrorlist is requested + * Always render content of directory on disk for local root + * Cleanup server_capability_check more aggressively + * Add json format for listing content of directory + * Add experimental subtree feature + * Add experimental MIRRORCACHE_PROXY_URL variable + * Fix rendering when local folder has broken symbolic links + * Fix multiroot location in task folder_hashes_create + +- Improve packaging, the conf.env is now in /etc/mirrorcache/ + * Add script to generate and update cache.tar.xz + +------------------------------------------------------------------- +Fri Sep 10 17:17:16 UTC 2021 - Elisei Roca + +- Update to version 1.007: + * UI: Solidify branding into lesser files + * Remove SUSE branding, it is not needed here + * Allow multiple folders in MIRRORCACHE_ROOT (#186) + * Show filestamps when rendering files with RootLocal (#184) + * Render error for when metalink and mirrorlist when file is not in DB (#185) + * Add parameter MIRRORCACHE_VPN_PREFIX (#182) + * Change origin in metalink to show metalink url + * [doc] fix in README command for geoip modules + * Use hostname_vpn in capability checks + * Fix one more typo (#179) + +- Remove 0001-Fix-one-more-typo-179.patch - included in 1.007 + +------------------------------------------------------------------- +Mon Aug 16 12:52:04 UTC 2021 - Elisei Roca + +- add 0001-Fix-one-more-typo-179.patch + +------------------------------------------------------------------- +Thu Aug 12 11:42:34 UTC 2021 - andrii.nikitin@suse.com + +- Update to version 1.006: + * Fix a small but significant typo + * Tests: add rsync to Dockerfile + * Add metalink content-type and content-disposition + +- Update to version 1.005: + * Use hostname_vpn if defined for mirror scan + * Add support for requests over vpn + * Use the Digest::SHA builtin module (#175) + * Replace Sort::Naturally with Sort::Versions (#174) + * Do not show /download path in mirrorlist when TOP_FOLDERS is set + * Add mirrorcache-backstage-hashes.service + * Log no miss for mirrorlist requests from country outside of region of subsidiary + * Add test_local, test_docker and test_systemd to Makefile + * Add column mirrorlist to stat table + * Add concept of project + +- Update to version 1.006: + * Fix a small but significant typo + * Tests: add rsync to Dockerfile + * Add metalink content-type and content-disposition + +- Update to version 1.005: + * Use hostname_vpn if defined for mirror scan + * Add support for requests over vpn: + parameter MIRRORCACHE_REDIRECT_VPN and new column server.hostname_vpn + * Use the Digest::SHA builtin module + * Improve file sorting by using Sort::Versions instead of Sort::Naturally + +- Update to version 1.004: + * Do not show /download path in mirrorlist when TOP_FOLDERS is set + * Add mirrorcache-backstage-hashes.service + +- Update to version 1.003: + * Log no miss for mirrorlist requests from country outside of region of subsidiary + * Add test_local, test_docker and test_systemd to Makefile + * Add column mirrorlist to stat table + * Add concept of project + * Log miss if a mirror has newer file in pedantic mode + * Improve detecting unversioned files + * Show checksums in mirrorlist when available + * Dont redirect to subsidiary requests for media.1/media and repomd.xml + +- Update to version 1.002: + * Enforce pedantic mode unless file is versioned (#165) + * Fix stat reporting as miss when ROOT_COUNTRY is set + * Include subsidiaries in mirrorlist + * Add task which imports hashes from HEADQUARTER + +- Update to version 1.001: + * t: Fix test redirect from landing page when TOP_FOLDERS is set + * Verify size on mirrors for files with -Current suffix + * Fix redirect to https when mirror doesnt have it + +- Update to version 1.0: + * Add /rest/mystat 'My Statistics' menu item + * Schedule jobs from stat (#158) + * Fix Datamodule unsafe usage in async rendering + * Fix render greeting on landing page when TOP_FOLDERS is set + +- Update to version 0.9: + * UI: Add branding capability and footer info + * Render landing page if TOP_FOLDERS is set + * Truncate country code to two letters + * Log trailing slash in stat + * Fix double stats when no mirror in country + * t: Replace repo in docker to openSUSE:infrastructure + * salt: switch test to use 15.3 + +- Update to version 0.8: + * Fix remote render dir when not in DB + * Add mirrors list page, reference it in file listing 'Metadata' column + * UI: Show title on every page + * Do not require /downloads path when TOP_FOLDERS is set + * Fix error "time_bad_sig: Return_to signature is not valid." during openid login + * Optimize /rest/stat (#148) + * Fix url for task MirrorCheckFromStat when metalink + * Add country region everywhere + * Performance improvements + * Set $app->secrets early + * dist: remove obsolete parts in salt + +- Update to version 0.7: + * Collect file hashes when HASHES_COLLECT is set + * Collect file stats in RootLocal + * Reduce permanent tasks' guard lifespan + * Improve file sorting in UI + * Switch to colspan instead of multiple + * Use POST when logging out + * Use only class "name" for content that is too long + * Fix wrong redirect when no mirror for media.1/media + * Fix filtering audit logs by id + +- Update to version 0.6: + * Permanently set MOJO_LISTEN to http://*:__port + * docs: update guide on running docker tests + * dist: rework creation of mirrorcache user and its home dir (#127) + * Add job mirror_scan_from_stat to ensure quality of responses + * t: Rename folder environs -> environ + * t: Switch to the new reworked environ framework + * Consider 403 code as server is reachable + * Add MIRRORCACHE_AUTH_URL, fake auth for testing + * Add audit log (#122) + * ui: css wrap only filename + +- Update to version 0.5: + * Adjust github address to openSUSE + * Obsolete MIRRORCACHE_FALLBACK_REDIRECT in favor of MIRRORCACHE_REDIRECT + +- Update to version 0.4: + * Add favicon and initial version of logo + * Support nginx file origin with autoindex + +- Update to version 0.3: + * Remove column db_sync_for_country from /app/folder + +- Update to version 0.2: + * Introduce timeout of folder rescan per country + * Also dont scan every country when scan of two countries was requested + +- Update to version 0.1: + * Auto deploy db on start diff --git a/MirrorCache.obsinfo b/MirrorCache.obsinfo new file mode 100644 index 0000000..d93650c --- /dev/null +++ b/MirrorCache.obsinfo @@ -0,0 +1,4 @@ +name: MirrorCache +version: 1.089 +mtime: 1734088621 +commit: 1bb100fccac4537ab3f0950e158bfe3b911d0691 diff --git a/MirrorCache.spec b/MirrorCache.spec new file mode 100644 index 0000000..11ff8fb --- /dev/null +++ b/MirrorCache.spec @@ -0,0 +1,103 @@ +# +# spec file for package MirrorCache +# +# Copyright (c) 2021 SUSE LLC +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via https://bugs.opensuse.org/ +# + + +%define mirrorcache_services_restart mirrorcache.service mirrorcache-backstage.service mirrorcache-backstage-hashes.service mirrorcache-subtree.service +%define mirrorcache_services %{mirrorcache_services_restart} mirrorcache-hypnotoad.service +%define assetpack_requires perl(CSS::Minifier::XS) >= 0.01 perl(JavaScript::Minifier::XS) >= 0.11 perl(Mojolicious::Plugin::AssetPack) >= 1.36 perl(IO::Socket::SSL) +%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4 perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801 perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece) perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg) perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base) perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg) perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands) perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX) perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base) perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA) perl(Config::IniFiles) +%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow sysuser-tools +Name: MirrorCache +Version: 1.089 +Release: 0 +Summary: WebApp to redirect and manage mirrors +License: GPL-2.0-or-later +Group: Productivity/Networking/Web/Servers +URL: https://github.com/openSUSE/MirrorCache +Source0: %{name}-%{version}.tar.xz +Source1: cache.tar.xz +Source2: %{name}-user.conf +Source3: %{name}-tmpfilesd.conf +# use update-cache (or tools/generate-packed-assets) to generate/update cache.tar.xz +Source101: update-cache.sh +BuildRequires: %{build_requires} +Requires: %{main_requires} +Requires: perl(Minion) >= 10.0 +BuildArch: noarch +%sysusers_requires + +%description +Mirror redirector web service, which automatically scans the main server and mirrors + +%prep +%setup -q -a1 + +%build +# make {?_smp_mflags} +%sysusers_generate_pre %{SOURCE2} %{name} + +%check + +%install +%make_install +# DEST_DIR={_datadir} +mkdir -p %{buildroot}%{_sbindir} +ln -s ../sbin/service %{buildroot}%{_sbindir}/rcmirrorcache +ln -s ../sbin/service %{buildroot}%{_sbindir}/rcmirrorcache-hypnotoad +ln -s ../sbin/service %{buildroot}%{_sbindir}/rcmirrorcache-backstage +ln -s ../sbin/service %{buildroot}%{_sbindir}/rcmirrorcache-backstage-hashes +ln -s ../sbin/service %{buildroot}%{_sbindir}/rcmirrorcache-subtree +install -D -m 0644 %{SOURCE2} %{buildroot}%{_sysusersdir}/%{name}.conf +install -D -m 0644 %{SOURCE3} %{buildroot}%{_tmpfilesdir}/%{name}.conf + +%pre -f %{name}.pre +%service_add_pre %{mirrorcache_services} + +%post +%tmpfiles_create %{_tmpfilesdir}/%{name}.conf +%service_add_post %{mirrorcache_services} + +%preun +%service_del_preun %{mirrorcache_services} + +%postun +%service_del_postun %{mirrorcache_services_restart} +%service_del_postun_without_restart mirrorcache-hypnotoad.service + +%files +%doc README.md +%license LICENSE +%{_sbindir}/rcmirrorcache +%{_sbindir}/rcmirrorcache-hypnotoad +%{_sbindir}/rcmirrorcache-backstage +%{_sbindir}/rcmirrorcache-backstage-hashes +%{_sbindir}/rcmirrorcache-subtree +%{_sysusersdir}/%{name}.conf +%{_tmpfilesdir}/%{name}.conf +%config(noreplace) %attr(-,root,mirrorcache) %{_sysconfdir}/mirrorcache/ +%ghost %dir %attr(0750,mirrorcache,-) %{_localstatedir}/lib/mirrorcache/ +# init +%dir %{_unitdir} +%{_unitdir}/mirrorcache.service +%{_unitdir}/mirrorcache-hypnotoad.service +%{_unitdir}/mirrorcache-backstage.service +%{_unitdir}/mirrorcache-backstage-hashes.service +%{_unitdir}/mirrorcache-subtree.service +# web libs +%{_datadir}/mirrorcache + +%changelog diff --git a/_service b/_service new file mode 100644 index 0000000..155172b --- /dev/null +++ b/_service @@ -0,0 +1,20 @@ + + + dist/rpm/MirrorCache.spec + dist/rpm/MirrorCache-user.conf + dist/rpm/MirrorCache-tmpfilesd.conf + dist/rpm/update-cache.sh + https://github.com/openSUSE/MirrorCache + @PARENT_TAG@ + @PARENT_TAG@ + git + enable + Andrii Nikitin <andrii.nikitin@suse.com> + + + + + xz + *.tar + + diff --git a/cache.tar.xz b/cache.tar.xz new file mode 100644 index 0000000..b276d7f --- /dev/null +++ b/cache.tar.xz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4878b0b4be51b433961c07cd1be45efd870b36caf9103a1abe6513c9602b876 +size 1338126 diff --git a/update-cache.sh b/update-cache.sh new file mode 100644 index 0000000..ce19fbe --- /dev/null +++ b/update-cache.sh @@ -0,0 +1,24 @@ +#!/usr/bin/bash + +# requirements: +# bzip2 cpio perl-IO-Socket-SSL perl-Mojolicious +# perl-Mojolicious-Plugin-AssetPack ruby3.3-rubygem-sass + +set -e + +export LC_ALL='en_US.UTF-8' +export LANG='en_US.UTF-8' + +mkdir -p MirrorCache-update-cache +rm -rf MirrorCache-update-cache/* +pushd MirrorCache-update-cache + +cpio -id < ../MirrorCache-*.obscpio +pushd MirrorCache-* + +./tools/generate-packed-assets +tar cvjf ../../cache.tar.xz assets/cache assets/assetpack.db + +popd +popd +rm -rf MirrorCache-update-cache/*