Skip to content

Fix #14688 (test-my-pr.py: run against the debian ftp server directly) #8491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danmar merged 4 commits into from
Apr 25, 2026
Merged

Fix #14688 (test-my-pr.py: run against the debian ftp server directly) #8491

Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
da02450
test
danmar Apr 25, 2026
7488035
remove unused import time
danmar Apr 25, 2026
4075c7d
Update tools/test-my-pr.py
danmar Apr 25, 2026
35632f1
args.p will be 256 by default
danmar Apr 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 114 additions & 24 deletions tools/test-my-pr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@
import donate_cpu_lib as lib
import argparse
import glob
import gzip
import natsort
import os
import sys
import random
import re
import requests
import subprocess


Expand All @@ -20,6 +24,100 @@ def format_float(a, b=1):
return 'N/A'


def ftp_get(url):
try:
response = requests.get(url, timeout=300)
if response.ok:
Comment thread
danmar marked this conversation as resolved.
Outdated
return response.content
except Exception:
pass
Comment thread
danmar marked this conversation as resolved.
Outdated
return None


def latestvername(names):
s = natsort.natsorted(names, key=lambda x: x[x.index('_')+1:x.index('.orig.tar')])
return s[-1]


def getpackages():
debian = 'https://ftp.debian.org/debian/'

data = ftp_get(debian + 'ls-lR.gz')
if data is None:
print('Failed to fetch ls-lR.gz')
sys.exit(1)

lines = gzip.decompress(data).decode('utf-8', errors='replace').splitlines()

Comment thread
danmar marked this conversation as resolved.
# Example content in ls-lR:
#./pool/main/0/0xffff:
#total 1452
#-rw-r--r-- 2 dak debadmin 6524 Dec 25 2016 0xffff_0.7-2.debian.tar.xz
#-rw-r--r-- 2 dak debadmin 1791 Dec 25 2016 0xffff_0.7-2.dsc
#-rw-r--r-- 2 dak debadmin 57168 Dec 25 2016 0xffff_0.7-2_amd64.deb
#-rw-r--r-- 2 dak debadmin 48578 Dec 26 2016 0xffff_0.7-2_arm64.deb
#-rw-r--r-- 2 dak debadmin 56730 Dec 26 2016 0xffff_0.7-2_armel.deb
#-rw-r--r-- 2 dak debadmin 57296 Dec 26 2016 0xffff_0.7-2_armhf.deb
#-rw-r--r-- 2 dak debadmin 60254 Dec 26 2016 0xffff_0.7-2_i386.deb
#-rw-r--r-- 2 dak debadmin 53130 Dec 26 2016 0xffff_0.7-2_mips.deb
#-rw-r--r-- 2 dak debadmin 52542 Dec 26 2016 0xffff_0.7-2_mips64el.deb
#-rw-r--r-- 2 dak debadmin 53712 Dec 26 2016 0xffff_0.7-2_mipsel.deb
#-rw-r--r-- 2 dak debadmin 51908 Dec 26 2016 0xffff_0.7-2_ppc64el.deb
#-rw-r--r-- 2 dak debadmin 53548 Dec 26 2016 0xffff_0.7-2_s390x.deb
#-rw-r--r-- 2 dak debadmin 65248 Dec 25 2016 0xffff_0.7.orig.tar.gz
#-rw-r--r-- 2 dak debadmin 6884 Jul 19 19:08 0xffff_0.8-1.debian.tar.xz
#-rw-r--r-- 2 dak debadmin 1807 Jul 19 19:08 0xffff_0.8-1.dsc
#-rw-r--r-- 2 dak debadmin 58908 Jul 19 19:08 0xffff_0.8-1_amd64.deb
#-rw-r--r-- 2 dak debadmin 51340 Jul 19 19:58 0xffff_0.8-1_arm64.deb
#-rw-r--r-- 2 dak debadmin 57612 Jul 19 20:13 0xffff_0.8-1_armel.deb
#-rw-r--r-- 2 dak debadmin 58584 Jul 19 19:58 0xffff_0.8-1_armhf.deb
#-rw-r--r-- 2 dak debadmin 57544 Jul 19 20:23 0xffff_0.8-1_hurd-i386.deb
#-rw-r--r-- 2 dak debadmin 62048 Jul 19 23:54 0xffff_0.8-1_i386.deb
#-rw-r--r-- 2 dak debadmin 55080 Jul 23 19:07 0xffff_0.8-1_kfreebsd-amd64.deb
#-rw-r--r-- 2 dak debadmin 58392 Jul 23 19:07 0xffff_0.8-1_kfreebsd-i386.deb
#-rw-r--r-- 2 dak debadmin 54144 Jul 19 22:28 0xffff_0.8-1_mips.deb
#-rw-r--r-- 2 dak debadmin 53648 Jul 20 00:56 0xffff_0.8-1_mips64el.deb
#-rw-r--r-- 2 dak debadmin 54740 Jul 19 22:58 0xffff_0.8-1_mipsel.deb
#-rw-r--r-- 2 dak debadmin 57424 Jul 19 19:58 0xffff_0.8-1_ppc64el.deb
#-rw-r--r-- 2 dak debadmin 53764 Jul 19 22:28 0xffff_0.8-1_s390x.deb
#-rw-r--r-- 2 dak debadmin 64504 Jul 19 19:08 0xffff_0.8.orig.tar.gz
#

path = None
previous_path = ''
archives = []
filename = None
filenames = []
for line in lines:
line = line.strip()
if len(line) < 4:
if filename:
res1 = re.match(r'(.*)-[0-9.]+$', path)
if res1 is None:
res1 = re.match(r'(.*)[-.][0-9.]+$', path)
res2 = re.match(r'(.*)-[0-9.]+$', previous_path)
if res2 is None:
res2 = re.match(r'(.*)[-.][0-9.]+$', previous_path)
if res1 is None or res2 is None or res1.group(1) != res2.group(1):
archives.append(path + '/' + latestvername(filenames))
else:
archives[-1] = path + '/' + latestvername(filenames)
if path:
previous_path = path
path = None
filename = None
filenames = []
elif line.startswith('./pool/main/'):
path = debian + line[2:-1]
elif path and line.endswith(('.orig.tar.gz', '.orig.tar.bz2', '.orig.tar.xz')):
Comment thread
danmar marked this conversation as resolved.
filename = line[1 + line.rfind(' '):]
filenames.append(filename)

Comment thread
danmar marked this conversation as resolved.
return archives




if __name__ == "__main__":
__my_script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
__work_path = os.path.expanduser(os.path.join('~', 'cppcheck-' + __my_script_name + '-workfolder'))
Expand All @@ -40,6 +138,19 @@ def format_float(a, b=1):

print(args)

if args.packages_path:
# You can download packages using daca2-download.py
args.packages = glob.glob(os.path.join(args.packages_path, '*.tar.xz'))
random.shuffle(args.packages)
elif args.packages is None:
args.packages = getpackages()
random.shuffle(args.packages)

if args.p is None and args.packages:
args.p = len(args.packages)

Comment thread
danmar marked this conversation as resolved.
Outdated
print('\n'.join(args.packages[:20]))

if not lib.check_requirements():
print("Error: Check requirements")
sys.exit(1)
Expand Down Expand Up @@ -100,35 +211,14 @@ def format_float(a, b=1):
print('Failed to compile your version of Cppcheck')
sys.exit(1)

if args.packages_path:
# You can download packages using daca2-download.py
args.packages = glob.glob(os.path.join(args.packages_path, '*.tar.xz'))
args.p = len(args.packages)
packages_idxs = list(range(args.p))
random.shuffle(packages_idxs)
elif args.packages:
args.p = len(args.packages)
packages_idxs = []
else:
packages_count = lib.get_packages_count()
if not packages_count:
print("network or server might be temporarily down..")
sys.exit(1)

packages_idxs = list(range(packages_count))
random.shuffle(packages_idxs)

packages_processed = 0
crashes = []
timeouts = []

while (packages_processed < args.p and len(packages_idxs) > 0) or args.packages:
if args.packages:
package = args.packages.pop()
else:
package = lib.get_package(packages_idxs.pop())
while packages_processed < args.p and args.packages:
package = args.packages.pop()

if package.startswith('ftp://') or package.startswith('http://'):
if package.startswith('ftp://') or package.startswith('https://'):
tgz = lib.download_package(work_path, package, None)
Comment thread
danmar marked this conversation as resolved.
if tgz is None:
print("No package downloaded")
Expand Down
Loading