From patchwork Thu Jun 4 14:18:10 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 53DFC1BC434C for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 01/16] parsemail: wrap parse_mail() in a single transaction Date: Thu, 4 Jun 2026 16:18:10 +0200 Message-ID: <20260604141826.2998337-2-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 98 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Wrap the entire parse_mail() call in transaction.atomic() so that all database writes from email parsing run inside a single transaction. The existing transaction.atomic() blocks inside parse_mail() become savepoints within this outer transaction. The series deduplication retry logic continues to work since savepoint rollbacks are scoped to their own savepoint. This also ensures that any on_commit() callbacks registered by signal handlers only fire after the full email has been parsed and all patch/series associations are committed. Signed-off-by: Robin Jarry --- patchwork/management/commands/parsemail.py | 4 +++- .../notes/parsemail-transaction-d4e5f6g7h8i9j0k1.yaml | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/parsemail-transaction-d4e5f6g7h8i9j0k1.yaml diff --git a/patchwork/management/commands/parsemail.py b/patchwork/management/commands/parsemail.py index bcb257fe9714..2f90047a991b 100644 --- a/patchwork/management/commands/parsemail.py +++ b/patchwork/management/commands/parsemail.py @@ -8,6 +8,7 @@ import logging import sys from django.core.management import base +from django.db import transaction from patchwork.parser import parse_mail from patchwork.parser import DuplicateMailError @@ -57,7 +58,8 @@ class Command(base.BaseCommand): # broken email (ValueError): 1 (this could be noisy, if it's an issue # we could use a different return code) try: - result = parse_mail(mail, options['list_id']) + with transaction.atomic(): + result = parse_mail(mail, options['list_id']) if result is None: logger.warning('Nothing added to database') except DuplicateMailError as exc: diff --git a/releasenotes/notes/parsemail-transaction-d4e5f6g7h8i9j0k1.yaml b/releasenotes/notes/parsemail-transaction-d4e5f6g7h8i9j0k1.yaml new file mode 100644 index 000000000000..37ebb0573906 --- /dev/null +++ b/releasenotes/notes/parsemail-transaction-d4e5f6g7h8i9j0k1.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Wrap the entire parse_mail() function in a single database + transaction to prevent partial state from being visible to + concurrent readers. From patchwork Thu Jun 4 14:18:11 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 6A1441BC434D for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 02/16] parsemail: fix SeriesReference race with concurrent delivery Date: Thu, 4 Jun 2026 16:18:11 +0200 Message-ID: <20260604141826.2998337-3-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 99 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit When multiple parsemail processes run in parallel (e.g. postfix delivering several messages from the same series at once), two processes can try to create a SeriesReference for the same msgid simultaneously. The second one fails with an IntegrityError: django.db.utils.IntegrityError: duplicate key value violates unique constraint "patchwork_seriesreference_project_id_msgid_..." DETAIL: Key (project_id, msgid)=(2, <...>) already exists. This can result in incomplete series that never reach the "received_all" state because the failed parsemail invocation prevents one of the patches from being recorded. The existing get/create pattern has a classic TOCTOU race: the get succeeds (no reference found), but by the time create runs, another process has already inserted the row. Replace both the try/get/ except/create block and the bare create call with get_or_create which handles the race atomically at the database level. Signed-off-by: Robin Jarry --- patchwork/parser.py | 30 +++++++++---------- .../parsemail-race-fix-e5f6g7h8i9j0k1l2.yaml | 5 ++++ 2 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 releasenotes/notes/parsemail-race-fix-e5f6g7h8i9j0k1l2.yaml diff --git a/patchwork/parser.py b/patchwork/parser.py index c33ada8d7d42..fafa47ba9e78 100644 --- a/patchwork/parser.py +++ b/patchwork/parser.py @@ -1336,20 +1336,18 @@ def parse_mail(mail, list_id=None): # later one. for ref in refs + [msgid]: ref = ref[:255] - # we don't want duplicates - try: - # we could have a ref to a previous series. - # (For example, a series sent in reply to - # another series.) That should not create a - # series ref for this series, so check for the - # msg-id only, not the msg-id/series pair. - SeriesReference.objects.get( - msgid=ref, project=project - ) - except SeriesReference.DoesNotExist: - SeriesReference.objects.create( - msgid=ref, project=project, series=series - ) + # We could have a ref to a previous series. (For + # example, a series sent in reply to another + # series.) That should not create a series ref for + # this series, so check for the msg-id only, not + # the msg-id/series pair. Use get_or_create to + # avoid races when multiple parsemail processes run + # in parallel. + SeriesReference.objects.get_or_create( + msgid=ref, + project=project, + defaults={'series': series}, + ) # attempt to pull the series in again, raising an # exception if we lost the race when creating a series @@ -1436,8 +1434,8 @@ def parse_mail(mail, list_id=None): # we don't save the in-reply-to or references fields # for a cover letter, as they can't refer to the same # series - SeriesReference.objects.create( - msgid=msgid, project=project, series=series + SeriesReference.objects.get_or_create( + msgid=msgid, project=project, defaults={'series': series} ) with transaction.atomic(): diff --git a/releasenotes/notes/parsemail-race-fix-e5f6g7h8i9j0k1l2.yaml b/releasenotes/notes/parsemail-race-fix-e5f6g7h8i9j0k1l2.yaml new file mode 100644 index 000000000000..f0869819538b --- /dev/null +++ b/releasenotes/notes/parsemail-race-fix-e5f6g7h8i9j0k1l2.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix a race condition in SeriesReference creation during concurrent + email delivery. From patchwork Thu Jun 4 14:18:12 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 818431BC434E for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 03/16] series: add respin tracking for patch series versions Date: Thu, 4 Jun 2026 16:18:12 +0200 Message-ID: <20260604141826.2998337-4-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 100 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit When a new version of a patch series is received (e.g. [PATCH v3]), automatically link it to the previous version via a new previous_series foreign key on the Series model. This creates a version chain that can be walked in both directions to find all versions of a series. The matching heuristic works in two tiers. First, In-Reply-To and References headers are checked against SeriesReference entries to find a threading link to any earlier version of the series from the same submitter. Since respins may reply to any previous version (e.g. v3 replying to v1 instead of v2), the code walks the matched series' version chain to locate the direct predecessor (version N-1). If no match is found through references, fall back to comparing series names and individual patch subjects using SequenceMatcher with a 0.8 similarity threshold, filtered by same submitter and consecutive version number. A new per-project auto_supersede boolean (default off) allows automatically marking all patches of the previous series version as "Superseded" when a newer version is linked. The REST API exposes previous_series and next_series fields on the series endpoint (API v1.4). The web UI shows a "Versions" row on the patch and cover letter detail pages with links to all versions. Signed-off-by: Robin Jarry --- patchwork/api/series.py | 20 ++- .../migrations/0049_series_respin_tracking.py | 33 +++++ patchwork/models.py | 34 +++++ patchwork/parser.py | 139 ++++++++++++++++++ patchwork/templates/patchwork/submission.html | 15 ++ patchwork/tests/api/test_series.py | 2 +- patchwork/views/cover.py | 3 + patchwork/views/patch.py | 2 + ...ries-respin-tracking-c3d4e5f6g7h8i9j0.yaml | 16 ++ 9 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 patchwork/migrations/0049_series_respin_tracking.py create mode 100644 releasenotes/notes/series-respin-tracking-c3d4e5f6g7h8i9j0.yaml diff --git a/patchwork/api/series.py b/patchwork/api/series.py index c9f5f54bdbfd..2ad805eadd13 100644 --- a/patchwork/api/series.py +++ b/patchwork/api/series.py @@ -33,6 +33,12 @@ class SeriesSerializer(BaseHyperlinkedModelSerializer): dependents = HyperlinkedRelatedField( read_only=True, view_name='api-series-detail', many=True ) + previous_series = HyperlinkedRelatedField( + read_only=True, view_name='api-series-detail' + ) + next_series = HyperlinkedRelatedField( + read_only=True, view_name='api-series-detail', many=True + ) def get_web_url(self, instance): request = self.context.get('request') @@ -71,6 +77,8 @@ class SeriesSerializer(BaseHyperlinkedModelSerializer): 'patches', 'dependencies', 'dependents', + 'previous_series', + 'next_series', ) read_only_fields = ( 'date', @@ -83,10 +91,17 @@ class SeriesSerializer(BaseHyperlinkedModelSerializer): 'patches', 'dependencies', 'dependents', + 'previous_series', + 'next_series', ) versioned_fields = { '1.1': ('web_url',), - '1.4': ('dependencies', 'dependents'), + '1.4': ( + 'dependencies', + 'dependents', + 'previous_series', + 'next_series', + ), } extra_kwargs = { 'url': {'view_name': 'api-series-detail'}, @@ -105,8 +120,9 @@ class SeriesMixin(object): 'cover_letter__project', 'dependencies', 'dependents', + 'next_series', ) - .select_related('submitter', 'project') + .select_related('submitter', 'project', 'previous_series') ) diff --git a/patchwork/migrations/0049_series_respin_tracking.py b/patchwork/migrations/0049_series_respin_tracking.py new file mode 100644 index 000000000000..18e5ddba915c --- /dev/null +++ b/patchwork/migrations/0049_series_respin_tracking.py @@ -0,0 +1,33 @@ +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('patchwork', '0048_series_dependencies'), + ] + + operations = [ + migrations.AddField( + model_name='series', + name='previous_series', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='next_series', + to='patchwork.series', + ), + ), + migrations.AddField( + model_name='project', + name='auto_supersede', + field=models.BooleanField( + default=False, + help_text=( + 'Automatically mark patches of previous series versions ' + 'as superseded when a new version is received.' + ), + ), + ), + ] diff --git a/patchwork/models.py b/patchwork/models.py index ae2f4a6dcbaa..387490a7094f 100644 --- a/patchwork/models.py +++ b/patchwork/models.py @@ -104,6 +104,11 @@ class Project(models.Model): default=False, help_text='Enable dependency tracking for patches and cover letters.', ) + auto_supersede = models.BooleanField( + default=False, + help_text='Automatically mark patches of previous series versions ' + 'as superseded when a new version is received.', + ) use_tags = models.BooleanField(default=True) def is_editable(self, user): @@ -858,6 +863,15 @@ class Series(FilenameMixin, models.Model): related_query_name='dependent', ) + # respin tracking + previous_series = models.ForeignKey( + 'self', + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name='next_series', + ) + # metadata name = models.CharField( max_length=255, @@ -967,6 +981,26 @@ class Series(FilenameMixin, models.Model): return patch + def get_version_chain(self): + """Return all versions of this series, oldest first.""" + chain = [self] + current = self + while current.previous_series_id: + current = current.previous_series + chain.append(current) + chain.reverse() + current = self + for newer in current.next_series.order_by('version'): + chain.append(newer) + # follow further respins from each newer version + queue = list(newer.next_series.order_by('version')) + while queue: + s = queue.pop(0) + if s not in chain: + chain.append(s) + queue.extend(s.next_series.order_by('version')) + return chain + def get_absolute_url(self): # TODO(stephenfin): We really need a proper series view return reverse( diff --git a/patchwork/parser.py b/patchwork/parser.py index fafa47ba9e78..13d043069944 100644 --- a/patchwork/parser.py +++ b/patchwork/parser.py @@ -6,6 +6,7 @@ import codecs import datetime from datetime import timezone +from difflib import SequenceMatcher from email.header import decode_header from email.header import make_header from email.utils import mktime_tz @@ -335,6 +336,126 @@ def find_series(project, mail, author): return _find_series_by_markers(project, mail, author) +def _strip_name_prefixes(name): + """Strip leading bracketed prefixes from a series or patch name.""" + if not name: + return '' + prefix_re = re.compile(r'^\[([^\]]*)\]\s*') + return prefix_re.sub('', name).strip() + + +def _name_similarity(a, b): + """Return similarity ratio between two stripped names.""" + a = _strip_name_prefixes(a) + b = _strip_name_prefixes(b) + if not a or not b: + return 0.0 + return SequenceMatcher(None, a.lower(), b.lower()).ratio() + + +def _find_version_in_chain(series, target_version): + """Walk a series' version chain to find a specific version.""" + # walk backward + current = series + while current: + if current.version == target_version: + return current + current = current.previous_series + + # walk forward + current = series + while current: + if current.version == target_version: + return current + nxt = current.next_series.order_by('version').first() + if nxt and nxt.id != current.id: + current = nxt + else: + break + + return None + + +def find_previous_series(project, series, refs): + """Find the previous version of a series for respin tracking. + + Uses a two-tier heuristic: first check mail references for a + direct threading link to a related series, then fall back to + name and patch subject similarity matching. + """ + if series.version <= 1: + return None + + prev_version = series.version - 1 + + # tier 1: check In-Reply-To / References for a link to any + # version of the same series from the same submitter, then walk + # the version chain to find version N-1 + for ref in refs: + try: + sr = SeriesReference.objects.get(msgid=ref[:255], project=project) + except SeriesReference.DoesNotExist: + continue + linked = sr.series + if linked.id == series.id: + continue + if linked.submitter != series.submitter: + continue + if linked.version >= series.version: + continue + prev = _find_version_in_chain(linked, prev_version) + if prev: + return prev + + # tier 2: name + submitter matching + candidates = Series.objects.filter( + project=project, + submitter=series.submitter, + version=prev_version, + ).order_by('-date') + + if not candidates.exists(): + return None + + best = None + best_score = 0.0 + + for candidate in candidates: + score = _name_similarity(series.name, candidate.name) + + # also check individual patch subjects for multi-patch series + new_patches = list(series.patches.values_list('name', flat=True)) + if new_patches: + old_patches = list( + candidate.patches.values_list('name', flat=True) + ) + if old_patches: + matches = 0 + for np in new_patches: + for op in old_patches: + if _name_similarity(np, op) >= 0.8: + matches += 1 + break + patch_ratio = matches / len(new_patches) + score = max(score, patch_ratio) + + if score >= 0.8 and score > best_score: + best = candidate + best_score = score + + return best + + +def _mark_previous_as_superseded(series): + """Mark all patches in a series as superseded.""" + try: + superseded = State.objects.get(slug='superseded') + except State.DoesNotExist: + logger.warning('No "superseded" state found, skipping auto-supersede') + return + series.patches.update(state=superseded) + + def split_from_header(from_header): name, email = (None, None) @@ -1389,6 +1510,15 @@ def parse_mail(mail, list_id=None): # parse patch dependencies series.add_dependencies(parse_depends_on(message)) + # link to the previous version of this series + if not series.previous_series and version > 1: + prev = find_previous_series(project, series, refs) + if prev: + series.previous_series = prev + series.save() + if project.auto_supersede: + _mark_previous_as_superseded(prev) + return patch elif x == 0: # (potential) cover letters # if refs are empty, it's implicitly a cover letter. If not, @@ -1460,6 +1590,15 @@ def parse_mail(mail, list_id=None): # entire patch series; parse them series.add_dependencies(parse_depends_on(message)) + # link to the previous version of this series + if not series.previous_series and version > 1: + prev = find_previous_series(project, series, refs) + if prev: + series.previous_series = prev + series.save() + if project.auto_supersede: + _mark_previous_as_superseded(prev) + return cover_letter # comments diff --git a/patchwork/templates/patchwork/submission.html b/patchwork/templates/patchwork/submission.html index cd74491c0e92..b02555eee6d5 100644 --- a/patchwork/templates/patchwork/submission.html +++ b/patchwork/templates/patchwork/submission.html @@ -97,6 +97,21 @@ {% endif %} +{% if version_chain and version_chain|length > 1 %} + + Versions + +{% for s in version_chain %} +{% if s.id == submission.series.id %} + v{{ s.version }} +{% else %} + v{{ s.version }} +{% endif %} +{% if not forloop.last %} | {% endif %} +{% endfor %} + + +{% endif %} {% if submission.related %} Related diff --git a/patchwork/tests/api/test_series.py b/patchwork/tests/api/test_series.py index 24d7d9a6899c..e38f6e75553e 100644 --- a/patchwork/tests/api/test_series.py +++ b/patchwork/tests/api/test_series.py @@ -197,7 +197,7 @@ class TestSeriesAPI(utils.APITestCase): create_cover(series=series_obj) create_patch(series=series_obj) - with self.assertNumQueries(8): + with self.assertNumQueries(9): self.client.get(self.api_url()) @utils.store_samples('series-detail') diff --git a/patchwork/views/cover.py b/patchwork/views/cover.py index 15013a89e1ce..2a501cfed572 100644 --- a/patchwork/views/cover.py +++ b/patchwork/views/cover.py @@ -47,6 +47,9 @@ def cover_detail(request, project_id, msgid): comments = comments.only('submitter', 'date', 'id', 'content', 'cover') context['comments'] = comments + if cover.series: + context['version_chain'] = cover.series.get_version_chain() + return render(request, 'patchwork/submission.html', context) diff --git a/patchwork/views/patch.py b/patchwork/views/patch.py index efe94f17c942..fd36bf98461d 100644 --- a/patchwork/views/patch.py +++ b/patchwork/views/patch.py @@ -124,6 +124,8 @@ def patch_detail(request, project_id, msgid): context['project'] = patch.project context['related_same_project'] = related_same_project context['related_different_project'] = related_different_project + if patch.series: + context['version_chain'] = patch.series.get_version_chain() if errors: context['errors'] = errors diff --git a/releasenotes/notes/series-respin-tracking-c3d4e5f6g7h8i9j0.yaml b/releasenotes/notes/series-respin-tracking-c3d4e5f6g7h8i9j0.yaml new file mode 100644 index 000000000000..1d3d99bbf517 --- /dev/null +++ b/releasenotes/notes/series-respin-tracking-c3d4e5f6g7h8i9j0.yaml @@ -0,0 +1,16 @@ +--- +features: + - | + Automatic respin tracking for patch series versions. When a new + version of a series is received (e.g. [PATCH v3]), patchwork links + it to the previous version using In-Reply-To/References headers or + name similarity matching. The patch detail page shows all versions + with links to navigate between them. + - | + New per-project ``auto_supersede`` option. When enabled, receiving a + new series version automatically marks all patches of the previous + version as superseded. +api: + - | + Add ``previous_series`` and ``next_series`` fields to the series + endpoint (v1.5) for navigating between series versions. From patchwork Thu Jun 4 14:18:13 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 992951BC434F for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 04/16] series: add metadata key-value store Date: Thu, 4 Jun 2026 16:18:13 +0200 Message-ID: <20260604141826.2998337-5-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 101 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add a SeriesMetadata model for attaching arbitrary key-value pairs to patch series. This will be used by forge backends to store references such as PR numbers and branch names (e.g. github_pr=42, github_branch=pwforge/1a2b/fix). The key field is indexed for fast lookups by metadata type. The value field is also indexed to support reverse lookups (e.g. finding which series is linked to a given PR number). Signed-off-by: Robin Jarry --- patchwork/admin.py | 9 ++++ patchwork/migrations/0050_series_metadata.py | 41 +++++++++++++++++++ patchwork/models.py | 17 ++++++++ patchwork/templates/patchwork/submission.html | 10 +++++ patchwork/templatetags/utils.py | 9 ++++ patchwork/tests/test_series.py | 39 ++++++++++++++++++ 6 files changed, 125 insertions(+) create mode 100644 patchwork/migrations/0050_series_metadata.py diff --git a/patchwork/admin.py b/patchwork/admin.py index d1c389a17b99..a26487850936 100644 --- a/patchwork/admin.py +++ b/patchwork/admin.py @@ -19,6 +19,7 @@ from patchwork.models import PatchRelation from patchwork.models import Person from patchwork.models import Project from patchwork.models import Series +from patchwork.models import SeriesMetadata from patchwork.models import SeriesReference from patchwork.models import State from patchwork.models import Tag @@ -181,6 +182,14 @@ class SeriesReferenceAdmin(admin.ModelAdmin): admin.site.register(SeriesReference, SeriesReferenceAdmin) +class SeriesMetadataAdmin(admin.ModelAdmin): + model = SeriesMetadata + list_display = ('series', 'key', 'value') + + +admin.site.register(SeriesMetadata, SeriesMetadataAdmin) + + class CheckAdmin(admin.ModelAdmin): list_display = ( 'patch', diff --git a/patchwork/migrations/0050_series_metadata.py b/patchwork/migrations/0050_series_metadata.py new file mode 100644 index 000000000000..5f6b2923fa9d --- /dev/null +++ b/patchwork/migrations/0050_series_metadata.py @@ -0,0 +1,41 @@ +# Generated by Django 5.1.15 on 2026-05-30 00:43 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('patchwork', '0049_series_respin_tracking'), + ] + + operations = [ + migrations.CreateModel( + name='SeriesMetadata', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('key', models.CharField(db_index=True, max_length=255)), + ('value', models.TextField(db_index=True, max_length=255)), + ( + 'series', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='metadata', + to='patchwork.series', + ), + ), + ], + options={ + 'ordering': ['key'], + 'unique_together': {('series', 'key')}, + }, + ), + ] diff --git a/patchwork/models.py b/patchwork/models.py index 387490a7094f..0b23b3bc495f 100644 --- a/patchwork/models.py +++ b/patchwork/models.py @@ -1042,6 +1042,23 @@ class SeriesReference(models.Model): unique_together = [('project', 'msgid')] +class SeriesMetadata(models.Model): + series = models.ForeignKey( + Series, + related_name='metadata', + on_delete=models.CASCADE, + ) + key = models.CharField(max_length=255, db_index=True) + value = models.TextField(max_length=255, db_index=True) + + def __str__(self): + return '%s=%s' % (self.key, self.value) + + class Meta: + unique_together = [('series', 'key')] + ordering = ['key'] + + class Bundle(models.Model): owner = models.ForeignKey( User, diff --git a/patchwork/templates/patchwork/submission.html b/patchwork/templates/patchwork/submission.html index b02555eee6d5..4cec9799b946 100644 --- a/patchwork/templates/patchwork/submission.html +++ b/patchwork/templates/patchwork/submission.html @@ -112,6 +112,16 @@ {% endif %} +{% if submission.series.metadata.all %} + + Metadata + +{% for entry in submission.series.metadata.all %} + {{ entry.key }}: {{ entry.value|metadata_value }}{% if not forloop.last %} | {% endif %} +{% endfor %} + + +{% endif %} {% if submission.related %} Related diff --git a/patchwork/templatetags/utils.py b/patchwork/templatetags/utils.py index 78c0aac80fc8..198fa0468e7c 100644 --- a/patchwork/templatetags/utils.py +++ b/patchwork/templatetags/utils.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later from django import template +from django.utils.html import format_html register = template.Library() @@ -16,3 +17,11 @@ def verbose_name_plural(obj): @register.simple_tag def is_editable(obj, user): return obj.is_editable(user) + + +@register.filter +def metadata_value(value): + s = str(value) + if s.startswith(('http://', 'https://')): + return format_html('{}', s, s) + return format_html('{}', s) diff --git a/patchwork/tests/test_series.py b/patchwork/tests/test_series.py index e5f60e3ae62c..dadf34e47a51 100644 --- a/patchwork/tests/test_series.py +++ b/patchwork/tests/test_series.py @@ -1081,3 +1081,42 @@ class SeriesDependencyTestCase(SeriesDependencyBase): self.assertEqual(series2.dependencies.count(), 1) self.assertEqual(series3.dependencies.count(), 2) self.assertEqual(series3.dependents.count(), 0) + + +class SeriesMetadataTest(TestCase): + def test_create_metadata(self): + series = utils.create_series() + meta = models.SeriesMetadata.objects.create( + series=series, key='github_pr', value='42' + ) + self.assertEqual(str(meta), 'github_pr=42') + + def test_unique_key_per_series(self): + from django.db import IntegrityError + + series = utils.create_series() + models.SeriesMetadata.objects.create( + series=series, key='github_pr', value='42' + ) + with self.assertRaises(IntegrityError): + models.SeriesMetadata.objects.create( + series=series, key='github_pr', value='99' + ) + + def test_lookup_by_key_and_value(self): + series = utils.create_series() + models.SeriesMetadata.objects.create( + series=series, key='github_pr', value='42' + ) + found = models.SeriesMetadata.objects.get(key='github_pr', value='42') + self.assertEqual(found.series, series) + + def test_multiple_keys(self): + series = utils.create_series() + models.SeriesMetadata.objects.create( + series=series, key='github_pr', value='42' + ) + models.SeriesMetadata.objects.create( + series=series, key='github_branch', value='pwforge/1a2b/fix' + ) + self.assertEqual(series.metadata.count(), 2) From patchwork Thu Jun 4 14:18:14 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id AF8C31BC4350 for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 05/16] forge: add per-project configuration model Date: Thu, 4 Jun 2026 16:18:14 +0200 Message-ID: <20260604141826.2998337-6-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 102 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add a ForgeConfig model that links a patchwork project to a forge repository. Each project can have one configuration per backend type, with independent sync direction flags (ml-to-forge and forge-to-ml) and a thread_respins toggle to control whether respin series are threaded under the original version. A from_email field specifies the sender address for forge-originated emails, with a sender_email property that falls back to DEFAULT_FROM_EMAIL when unset. A tabular inline on the Project admin page allows maintainers to configure forge coupling directly. Signed-off-by: Robin Jarry --- patchwork/admin.py | 15 ++++++++ patchwork/migrations/0051_forgeconfig.py | 46 +++++++++++++++++++++++ patchwork/models.py | 47 ++++++++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 patchwork/migrations/0051_forgeconfig.py diff --git a/patchwork/admin.py b/patchwork/admin.py index a26487850936..e138ad4f2f8e 100644 --- a/patchwork/admin.py +++ b/patchwork/admin.py @@ -13,6 +13,7 @@ from patchwork.models import Check from patchwork.models import Cover from patchwork.models import CoverComment from patchwork.models import DelegationRule +from patchwork.models import ForgeConfig from patchwork.models import Patch from patchwork.models import PatchComment from patchwork.models import PatchRelation @@ -45,10 +46,24 @@ class DelegationRuleInline(admin.TabularInline): fields = ('path', 'user', 'priority') +class ForgeConfigInline(admin.TabularInline): + model = ForgeConfig + fields = ( + 'backend', + 'repo', + 'from_email', + 'sync_ml_to_forge', + 'sync_forge_to_ml', + 'thread_respins', + ) + extra = 0 + + class ProjectAdmin(admin.ModelAdmin): list_display = ('name', 'linkname', 'listid', 'listemail') inlines = [ DelegationRuleInline, + ForgeConfigInline, ] diff --git a/patchwork/migrations/0051_forgeconfig.py b/patchwork/migrations/0051_forgeconfig.py new file mode 100644 index 000000000000..36f041d24021 --- /dev/null +++ b/patchwork/migrations/0051_forgeconfig.py @@ -0,0 +1,46 @@ +# Generated by Django 5.1.15 on 2026-05-30 00:32 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('patchwork', '0050_series_metadata'), + ] + + operations = [ + migrations.CreateModel( + name='ForgeConfig', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('backend', models.CharField(max_length=50)), + ('repo', models.CharField(max_length=255)), + ('from_email', models.CharField(default='', max_length=255)), + ('sync_ml_to_forge', models.BooleanField(default=True)), + ('sync_forge_to_ml', models.BooleanField(default=True)), + ('thread_respins', models.BooleanField(default=False)), + ( + 'project', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='forges', + to='patchwork.project', + ), + ), + ], + options={ + 'verbose_name': 'forge configuration', + 'verbose_name_plural': 'forge configurations', + 'unique_together': {('project', 'backend')}, + }, + ), + ] diff --git a/patchwork/models.py b/patchwork/models.py index 0b23b3bc495f..3f7974d92ccb 100644 --- a/patchwork/models.py +++ b/patchwork/models.py @@ -130,6 +130,53 @@ class Project(models.Model): ordering = ['linkname'] +class ForgeConfig(models.Model): + project = models.ForeignKey( + Project, related_name='forges', on_delete=models.CASCADE + ) + backend = models.CharField( + max_length=50, + help_text='Forge backend name (e.g. "github").', + ) + repo = models.CharField( + max_length=255, + help_text='Repository identifier (e.g. "owner/repo").', + ) + from_email = models.CharField( + max_length=255, + default='', + help_text='Sender address for forge-originated emails.', + ) + sync_ml_to_forge = models.BooleanField( + default=True, + help_text='Create forge PRs from mailing list patch series.', + ) + sync_forge_to_ml = models.BooleanField( + default=True, + help_text='Send patch emails from forge PRs to the mailing list.', + ) + thread_respins = models.BooleanField( + default=False, + help_text='Thread respin series as replies to the original version.', + ) + + @property + def sender_email(self): + """ + Return from_email if set, otherwise fall back to + DEFAULT_FROM_EMAIL. + """ + return self.from_email or settings.DEFAULT_FROM_EMAIL + + def __str__(self): + return '%s (%s)' % (self.repo, self.backend) + + class Meta: + verbose_name = 'forge configuration' + verbose_name_plural = 'forge configurations' + unique_together = [('project', 'backend')] + + class DelegationRule(models.Model): project = models.ForeignKey(Project, on_delete=models.CASCADE) user = models.ForeignKey( From patchwork Thu Jun 4 14:18:15 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id C6F141BC4351 for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 06/16] forge: add backend abstraction layer Date: Thu, 4 Jun 2026 16:18:15 +0200 Message-ID: <20260604141826.2998337-7-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 103 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add a forge integration framework that allows patchwork to receive webhooks from code forges. The abstraction is designed so that new forge backends can be added without changing the core logic. A ForgeBackend abstract base class defines the interface that each backend must implement: webhook signature verification, event parsing, and sync event processing. Backends register themselves at import time and are only loaded when explicitly listed in the FORGE_BACKENDS setting, keeping forge-specific dependencies entirely optional. ForgeConfig.clean() validates that the backend name matches a registered backend. Webhook requests are received at /webhook/forge// and the whole feature is gated behind the ENABLE_FORGE setting. Signed-off-by: Robin Jarry --- patchwork/forge/__init__.py | 175 ++++++++++++++++++++++++++++++++++++ patchwork/forge/urls.py | 16 ++++ patchwork/forge/views.py | 82 +++++++++++++++++ patchwork/models.py | 9 ++ patchwork/settings/base.py | 26 ++++++ patchwork/urls.py | 9 ++ 6 files changed, 317 insertions(+) create mode 100644 patchwork/forge/__init__.py create mode 100644 patchwork/forge/urls.py create mode 100644 patchwork/forge/views.py diff --git a/patchwork/forge/__init__.py b/patchwork/forge/__init__.py new file mode 100644 index 000000000000..1c236e4a97f2 --- /dev/null +++ b/patchwork/forge/__init__.py @@ -0,0 +1,175 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +""" +Forge integration framework. + +Provides an abstraction layer for synchronizing patch workflows between +patchwork and code forges (GitHub, GitLab, etc.). Each forge platform is +implemented as a backend that registers itself at import time and is only +loaded when listed in settings.FORGE_BACKENDS. + +Backends are responsible for webhook signature verification and event parsing. +Sync logic (creating patches from PRs, forwarding comments) is implemented +per-backend but can reuse generic utilities from patchwork.forge.sync. + +Forge-specific dependencies remain entirely optional: a backend module is never +imported unless explicitly enabled in settings. +""" + +import importlib +import logging +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from dataclasses import field + +from django.conf import settings + +logger = logging.getLogger(__name__) + + +@dataclass +class ForgeUser: + login: str = '' + name: str = '' + email: str = '' + + +@dataclass +class ReviewComment: + path: str = '' + diff_hunk: str = '' + body: str = '' + + +@dataclass +class CheckRun: + name: str = '' + status: str = '' + url: str = '' + description: str = '' + + +@dataclass +class ForgeEvent: + type: str = '' + repo_key: str = '' + pr_number: int = 0 + author: ForgeUser = field(default_factory=ForgeUser) + body: str = '' + + # review fields + review_id: int = 0 + review_state: str = '' + + # check fields + check_suite_id: int = 0 + check_name: str = '' + check_status: str = '' + check_url: str = '' + check_description: str = '' + + # pull request fields + pr_title: str = '' + pr_body: str = '' + pr_head: str = '' + pr_base: str = '' + pr_head_branch: str = '' + pr_action: str = '' + pr_before: str = '' + + +class ForgeBackend(ABC): + """ + Base class for forge backend implementations. + + Each backend handles webhook signature verification and event parsing for + a specific forge platform (GitHub, GitLab, etc.). Backends register + themselves at import time via register_backend(). + """ + + @abstractmethod + def verify_webhook_signature(self, body, headers, secret): + """ + Verify the HMAC signature of an incoming webhook request. + + Args: + body (bytes): Raw request body. + headers (dict): HTTP headers. + secret (str): Shared secret for signature verification. + If empty, verification is skipped. + + Returns: + True if the signature is valid or no secret is configured. + """ + raise NotImplementedError + + @abstractmethod + def parse_webhook_event(self, body, headers): + """ + Parse a webhook payload into a ForgeEvent. + + Args: + body (bytes): Raw request body. + headers (dict): HTTP headers. + + Returns: + A ForgeEvent instance, or None if the event should be ignored. + """ + raise NotImplementedError + + @abstractmethod + def process_webhook_event(self, forge_config, event): + """ + Handle a parsed webhook event for a given project. + + Called by the webhook view after signature verification, event + parsing and project routing. The backend decides which event + types to act on and what sync actions to perform. + """ + raise NotImplementedError + + @abstractmethod + def series_metadata(self, forge_config, event): + """ + Return a dict of SeriesMetadata key-value pairs to associate + with a series created from a forge event. + + Used after ingesting patches to link the series back to its + forge pull request and branch. + """ + raise NotImplementedError + + def get_auth(self, forge_config): + """ + Resolve authentication credentials for a forge config. + + Merges backend-level defaults from FORGE_AUTH[backend] with per-repo + overrides from FORGE_AUTH[backend]["repos"][repo]. + """ + backend_auth = settings.FORGE_AUTH.get(forge_config.backend, {}) + auth = {k: v for k, v in backend_auth.items() if k != 'repos'} + repo_overrides = backend_auth.get('repos', {}).get( + forge_config.repo, {} + ) + auth.update(repo_overrides) + return auth + + +_backends = {} + + +def register_backend(name, backend): + _backends[name] = backend + + +def get_backend(name): + return _backends.get(name) + + +def load_backends(): + for module_path in settings.FORGE_BACKENDS: + importlib.import_module(module_path) diff --git a/patchwork/forge/urls.py b/patchwork/forge/urls.py new file mode 100644 index 000000000000..ece29f5e59d1 --- /dev/null +++ b/patchwork/forge/urls.py @@ -0,0 +1,16 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +from django.urls import path + +from patchwork.forge import views + +urlpatterns = [ + path( + '/', + views.forge_webhook, + name='forge-webhook', + ), +] diff --git a/patchwork/forge/views.py b/patchwork/forge/views.py new file mode 100644 index 000000000000..e8d0db35c79b --- /dev/null +++ b/patchwork/forge/views.py @@ -0,0 +1,82 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import logging + +from django.conf import settings +from django.http import HttpResponse +from django.http import HttpResponseNotAllowed +from django.http import HttpResponseNotFound +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt + +from patchwork.forge import get_backend +from patchwork.models import ForgeConfig + +logger = logging.getLogger(__name__) + + +@csrf_exempt +def forge_webhook(request, backend_name): + """ + Receive and process a webhook from a forge backend. + + Verifies the request signature, parses the event payload and + dispatches it to the appropriate backend sync handler. + """ + if request.method != 'POST': + return HttpResponseNotAllowed(['POST']) + + backend = get_backend(backend_name) + if backend is None: + return HttpResponseNotFound() + + secret = settings.FORGE_WEBHOOK_SECRETS.get(backend_name, '') + if not backend.verify_webhook_signature( + request.body, request.headers, secret + ): + return HttpResponse(status=403) + + try: + event = backend.parse_webhook_event(request.body, request.headers) + except Exception: + logger.exception('failed to parse %s webhook', backend_name) + return HttpResponse(status=400) + + if event is None: + return JsonResponse({'status': 'ignored'}) + + try: + forge_config = ForgeConfig.objects.select_related('project').get( + backend=backend_name, repo=event.repo_key + ) + except ForgeConfig.DoesNotExist: + logger.warning( + '%s webhook: no project for repo %s', + backend_name, + event.repo_key, + ) + return JsonResponse({'status': 'unlinked'}) + + logger.info( + '%s webhook: type=%s repo=%s pr=%s project=%s', + backend_name, + event.type, + event.repo_key, + event.pr_number, + forge_config.project.linkname, + ) + + try: + backend.process_webhook_event(forge_config, event) + except Exception: + logger.exception( + 'failed to handle %s event for %s', + event.type, + forge_config.project.linkname, + ) + return HttpResponse(status=500) + + return JsonResponse({'status': 'ok'}) diff --git a/patchwork/models.py b/patchwork/models.py index 3f7974d92ccb..79e6259976ad 100644 --- a/patchwork/models.py +++ b/patchwork/models.py @@ -168,6 +168,15 @@ class ForgeConfig(models.Model): """ return self.from_email or settings.DEFAULT_FROM_EMAIL + def clean(self): + from patchwork.forge import get_backend + + backend = get_backend(self.backend) if self.backend else None + if self.backend and backend is None: + raise ValidationError( + {'backend': f'Unknown forge backend: {self.backend}'} + ) + def __str__(self): return '%s (%s)' % (self.repo, self.backend) diff --git a/patchwork/settings/base.py b/patchwork/settings/base.py index dccab6c36dba..e5bcbacf05dc 100644 --- a/patchwork/settings/base.py +++ b/patchwork/settings/base.py @@ -277,3 +277,29 @@ FORCE_HTTPS_LINKS = False # Set to True to hide admin details from the about page (/about) ADMINS_HIDE = False + +# Set to True to enable forge integration (webhook endpoints) +ENABLE_FORGE = False + +# List of forge backend modules to load (e.g. ['patchwork.forge.github']) +FORGE_BACKENDS = [] + +# Webhook secrets keyed by backend name (e.g. {'github': 'secret'}) +FORGE_WEBHOOK_SECRETS = {} + +# Authentication credentials keyed by backend name. Each backend +# defines its own auth fields. Per-repo overrides can be specified +# under a "repos" sub-dict. +# +# FORGE_AUTH = { +# "github": { +# "token": "ghp_default", # default for all repos +# "repos": { +# "owner/repo-a": { # per-repo override +# "app_id": 123456, +# "private_key_file": "/etc/patchwork/repo-a.pem", +# }, +# }, +# }, +# } +FORGE_AUTH = {} diff --git a/patchwork/urls.py b/patchwork/urls.py index 11cd8e7c152a..297119b8e816 100644 --- a/patchwork/urls.py +++ b/patchwork/urls.py @@ -366,6 +366,15 @@ if settings.ENABLE_REST_API: ] +if settings.ENABLE_FORGE: + from patchwork.forge import load_backends + + load_backends() + urlpatterns += [ + path('webhook/forge/', include('patchwork.forge.urls')), + ] + + # redirect from old urls if settings.COMPAT_REDIR: urlpatterns += [ From patchwork Thu Jun 4 14:18:16 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id DDBF61BC4352 for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 07/16] forge: add utilities for mailing-list sync Date: Thu, 4 Jun 2026 16:18:16 +0200 Message-ID: <20260604141826.2998337-8-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 104 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add generic utility functions reusable across forge backends: sanitize_pr_body() strips HTML comments and AI-generated sections from pull request descriptions. sender_identity() resolves a (name, email) tuple for forge users, falling back to the project's sender_email. find_series_by_pr() and next_version() query SeriesMetadata to locate previously synced series and determine respin version numbers. reply_to_msgid() finds the message-id to thread replies under. ingest_emails() parses an mbox and creates Series/Patch/Cover objects directly in the database via parse_mail(), then stores forge metadata on the series. send_emails() forwards raw patch bytes via SMTP preserving the original message format. Also add an ignore_hints parameter to parse_mail() so that X-Patchwork-Hint: ignore headers do not prevent direct ingestion. Signed-off-by: Robin Jarry --- patchwork/forge/util.py | 217 +++++++++++++++ patchwork/parser.py | 5 +- patchwork/tests/forge/__init__.py | 0 patchwork/tests/forge/test_util.py | 406 +++++++++++++++++++++++++++++ 4 files changed, 626 insertions(+), 2 deletions(-) create mode 100644 patchwork/forge/util.py create mode 100644 patchwork/tests/forge/__init__.py create mode 100644 patchwork/tests/forge/test_util.py diff --git a/patchwork/forge/util.py b/patchwork/forge/util.py new file mode 100644 index 000000000000..0793d368b7bd --- /dev/null +++ b/patchwork/forge/util.py @@ -0,0 +1,217 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +""" +Generic utilities for mailing-list synchronization. +""" + +import email +import io +import logging +import mailbox +import os +import re + +from django.core.mail import get_connection +from django.db import transaction + +from patchwork.models import Cover +from patchwork.models import Patch +from patchwork.models import Series +from patchwork.models import SeriesMetadata +from patchwork.parser import DuplicateMailError +from patchwork.parser import clean_header +from patchwork.parser import parse_mail + +logger = logging.getLogger(__name__) + +HTML_COMMENT_RE = re.compile(r'(?s)') + +AI_SECTION_HEADERS = [ + 'summary by coderabbit', + 'summary by copilot', + 'walkthrough', + 'generated by', +] + + +def sanitize_pr_body(body): + """ + Strip HTML comments and AI-generated sections from a pull request body + before using it as a cover letter description. + """ + if not body: + return '' + body = HTML_COMMENT_RE.sub('', body) + lines = body.split('\n') + result = [] + skip = False + for line in lines: + lower = line.strip().lower() + if lower.startswith('#'): + heading = lower.lstrip('# ') + for marker in AI_SECTION_HEADERS: + if heading.startswith(marker): + skip = True + break + if skip: + continue + skip = False + if not skip: + result.append(line) + return '\n'.join(result).strip() + + +def sender_identity(user, forge_config): + """ + Return a (name, email) tuple for a forge user. Falls back to the project's + sender_email when the user has no public email address. + """ + name = user.name or user.login + if user.email: + addr = user.email + else: + addr = email.utils.parseaddr(forge_config.sender_email)[1] + name += ' (via Patchwork)' + return name, addr + + +def find_series_by_pr(backend, forge_config, pr_number): + """ + Find all series linked to a pull request via SeriesMetadata, + ordered by most recent first. + """ + return ( + Series.objects.filter( + project=forge_config.project, + metadata__key=backend.meta_key_pr(), + metadata__value=backend.pr_ref(forge_config, pr_number), + ) + .select_related('cover_letter') + .order_by('version') + ) + + +def reply_to_msgid(series): + """ + Return the message-id to use as In-Reply-To when sending replies to a + series. Prefers the cover letter, falls back to first patch. + """ + if series.cover_letter: + return series.cover_letter.msgid + patches = list(series.patches.order_by('number')[:1]) + if patches: + return patches[0].msgid + return '' + + +def next_version(backend, forge_config, event): + """ + Determine the version number and threading info for a respin. + + Returns (version, in_reply_to, previous_ref) where version is the next + version number, in_reply_to is the message-id of the original series cover + letter, and previous_ref is the previous HEAD SHA for range-diff + generation. + """ + series = find_series_by_pr(backend, forge_config, event.pr_number) + first = series.first() + last = series.last() + + if not first or not last: + return 1, '', '' + + return last.version + 1, reply_to_msgid(first), event.pr_before + + +def bytes_to_mbox(buf): + """ + Create a mailbox.mbox object from bytes. + """ + # The builtin constructor only accepts file paths. Trick it by passing + # /dev/null and replace the opened file with BytesIO. + mbox = mailbox.mbox(os.devnull, create=False) + mbox._file.close() + mbox._file = io.BytesIO(buf) + return mbox + + +@transaction.atomic +def ingest_emails(mbox, backend, forge_config, event): + """ + Parse raw emails from a mailbox.mbox and create Series/Patch/Cover objects + in the database via parse_mail(). After ingestion, store forge metadata on + the series using backend.series_metadata(). + """ + list_id = forge_config.project.listid + series = None + + for msg in mbox: + try: + result = parse_mail(msg, list_id, ignore_hints=True) + except DuplicateMailError: + logger.warning( + 'patch already ingested: %s: %s', + clean_header(msg.get('Message-ID')), + clean_header(msg.get('Subject')), + ) + continue + except ValueError: + logger.exception( + 'failed to ingest patch: %s: %s', + clean_header(msg.get('Message-ID')), + clean_header(msg.get('Subject')), + ) + continue + + if isinstance(result, (Cover, Patch)): + series = result.series + + if series: + metadata = backend.series_metadata(forge_config, event) + for key, value in metadata.items(): + if value: + SeriesMetadata.objects.update_or_create( + series=series, + key=key, + defaults={'value': value}, + ) + + +def _msg_header_addresses(msg, *headers): + values = [] + for h in headers: + for v in msg.get_all(h, []): + txt = clean_header(v) + if txt: + values.append(txt) + + addrs = set() + for _, addr in email.utils.getaddresses(values): + addrs.add(addr) + return list(addrs) + + +def send_emails(mbox, forge_config): + """ + Send raw emails from a mailbox.mbox via SMTP. Reads Sender, From, To and Cc + addresses from the email headers. + """ + with get_connection(fail_silently=False) as conn: + for key, msg in mbox.iteritems(): + senders = _msg_header_addresses(msg, 'sender') + recipients = _msg_header_addresses(msg, 'from', 'to', 'cc') + logger.info( + 'sending patch: %s -> %s: %s', + ','.join(senders), + ','.join(recipients), + clean_header(msg.get('subject', '')), + ) + # XXX: only works if email backend is smtp + errs = conn.connection.sendmail( + senders[0], recipients, mbox.get_bytes(key) + ) + for rcpt, err in errs.items(): + logger.warning('send patch to %s failed: %s', rcpt, err) diff --git a/patchwork/parser.py b/patchwork/parser.py index 13d043069944..f3d454f96828 100644 --- a/patchwork/parser.py +++ b/patchwork/parser.py @@ -1310,12 +1310,13 @@ def find_comment_addressed_by_header(mail): return False if 'X-Patchwork-Action-Required' in mail else None -def parse_mail(mail, list_id=None): +def parse_mail(mail, list_id=None, ignore_hints=False): """Parse a mail and add to the database. Args: mail (`mbox.Mail`): Mail to parse and add. list_id (str): Mailing list ID + ignore_hint (bool): Ignore X-Patchwork-Hint headers. Returns: patch/cover letter/comment @@ -1338,7 +1339,7 @@ def parse_mail(mail, list_id=None): raise ValueError("Missing 'Message-Id' header") hint = clean_header(mail.get('X-Patchwork-Hint', '')) - if hint and hint.lower() == 'ignore': + if hint and hint.lower() == 'ignore' and not ignore_hints: logger.info("Ignoring email due to 'ignore' hint") return diff --git a/patchwork/tests/forge/__init__.py b/patchwork/tests/forge/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/patchwork/tests/forge/test_util.py b/patchwork/tests/forge/test_util.py new file mode 100644 index 000000000000..c5b18c855f82 --- /dev/null +++ b/patchwork/tests/forge/test_util.py @@ -0,0 +1,406 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +from unittest.mock import MagicMock +from unittest.mock import patch as mock_patch + +from django.test import TestCase + +from patchwork.forge import ForgeEvent +from patchwork.forge import ForgeUser +from patchwork.forge.util import bytes_to_mbox +from patchwork.forge.util import ingest_emails +from patchwork.forge.util import next_version +from patchwork.forge.util import reply_to_msgid +from patchwork.forge.util import sanitize_pr_body +from patchwork.forge.util import send_emails +from patchwork.forge.util import sender_identity +from patchwork.models import ForgeConfig +from patchwork.models import SeriesMetadata +from patchwork.tests.utils import create_cover +from patchwork.tests.utils import create_patches +from patchwork.tests.utils import create_project +from patchwork.tests.utils import create_series + + +class SanitizePRBodyTest(TestCase): + def test_strip_html_comments(self): + body = 'Hello\n\nWorld' + self.assertEqual(sanitize_pr_body(body), 'Hello\n\nWorld') + + def test_strip_multiline_html_comment(self): + body = 'Before\n\nAfter' + self.assertEqual(sanitize_pr_body(body), 'Before\n\nAfter') + + def test_strip_coderabbit_section(self): + body = ( + 'Real content\n\n' + '## Summary by CodeRabbit\n\n' + 'AI generated stuff\n' + 'more AI stuff\n' + ) + self.assertEqual(sanitize_pr_body(body), 'Real content') + + def test_strip_copilot_section(self): + body = 'Fix bug\n\n## Summary by Copilot\n\nAI stuff' + self.assertEqual(sanitize_pr_body(body), 'Fix bug') + + def test_strip_walkthrough_section(self): + body = 'Real\n\n## Walkthrough\n\nAI stuff' + self.assertEqual(sanitize_pr_body(body), 'Real') + + def test_preserve_normal_headings(self): + body = '## Description\n\nThis is fine\n\n## Notes\n\nAlso fine' + self.assertEqual(sanitize_pr_body(body), body) + + def test_empty_body(self): + self.assertEqual(sanitize_pr_body(''), '') + + def test_none_body(self): + self.assertEqual(sanitize_pr_body(None), '') + + +class SenderIdentityTest(TestCase): + def test_user_with_name_and_email(self): + user = ForgeUser(login='octocat', name='Octo Cat', email='o@c.com') + config = ForgeConfig(from_email='pw@example.com') + self.assertEqual( + sender_identity(user, config), ('Octo Cat', 'o@c.com') + ) + + def test_user_with_email_only(self): + user = ForgeUser(login='octocat', name='', email='o@c.com') + config = ForgeConfig(from_email='pw@example.com') + self.assertEqual( + sender_identity(user, config), + ('octocat', 'o@c.com'), + ) + + def test_user_without_email(self): + user = ForgeUser(login='octocat', name='Octo Cat', email='') + config = ForgeConfig(from_email='pw@example.com') + self.assertEqual( + sender_identity(user, config), + ('Octo Cat (via Patchwork)', 'pw@example.com'), + ) + + def test_user_without_email_fallback(self): + user = ForgeUser(login='octocat', name='', email='') + config = ForgeConfig(from_email='') + name, addr = sender_identity(user, config) + self.assertEqual(name, 'octocat (via Patchwork)') + self.assertTrue(addr) + + +class ReplyToMsgidTest(TestCase): + def test_series_with_cover_letter(self): + project = create_project() + series = create_series(project=project) + cover = create_cover(series=series) + self.assertEqual(reply_to_msgid(series), cover.msgid) + + def test_series_without_cover_letter(self): + project = create_project() + series = create_series(project=project) + patches = create_patches(count=1, series=series) + self.assertEqual(reply_to_msgid(series), patches[0].msgid) + + def test_empty_series(self): + series = create_series() + self.assertEqual(reply_to_msgid(series), '') + + +class NextVersionTest(TestCase): + def test_no_previous_series(self): + project = create_project() + backend = MagicMock() + backend.pr_ref.return_value = 'https://github.com/o/r/pull/1' + backend.meta_key_pr.return_value = 'github_pr' + forge_config = MagicMock() + forge_config.project = project + event = ForgeEvent(pr_number=1, pr_before='abc123') + + version, in_reply_to, previous_ref = next_version( + backend, forge_config, event + ) + self.assertEqual(version, 1) + self.assertEqual(in_reply_to, '') + self.assertEqual(previous_ref, '') + + def test_with_previous_series(self): + project = create_project() + series_v1 = create_series(project=project, version=1) + cover_v1 = create_cover(series=series_v1) + pr_ref = 'https://github.com/o/r/pull/42' + SeriesMetadata.objects.create( + series=series_v1, key='github_pr', value=pr_ref + ) + + backend = MagicMock() + backend.pr_ref.return_value = pr_ref + backend.meta_key_pr.return_value = 'github_pr' + forge_config = MagicMock() + forge_config.project = project + event = ForgeEvent(pr_number=42, pr_before='def456') + + version, in_reply_to, previous_ref = next_version( + backend, forge_config, event + ) + self.assertEqual(version, 2) + self.assertEqual(in_reply_to, cover_v1.msgid) + self.assertEqual(previous_ref, 'def456') + + def test_with_multiple_versions(self): + project = create_project() + series_v1 = create_series(project=project, version=1) + cover_v1 = create_cover(series=series_v1) + series_v2 = create_series(project=project, version=2) + create_cover(series=series_v2) + pr_ref = 'https://github.com/o/r/pull/42' + SeriesMetadata.objects.create( + series=series_v1, key='github_pr', value=pr_ref + ) + SeriesMetadata.objects.create( + series=series_v2, key='github_pr', value=pr_ref + ) + + backend = MagicMock() + backend.pr_ref.return_value = pr_ref + backend.meta_key_pr.return_value = 'github_pr' + forge_config = MagicMock() + forge_config.project = project + event = ForgeEvent(pr_number=42, pr_before='ghi789') + + version, in_reply_to, previous_ref = next_version( + backend, forge_config, event + ) + self.assertEqual(version, 3) + self.assertEqual(in_reply_to, cover_v1.msgid) + self.assertEqual(previous_ref, 'ghi789') + + +class BytesToMboxTest(TestCase): + MBOX_DATA = ( + b'From nobody Thu Jan 1 00:00:00 1970\n' + b'From: Test \n' + b'Subject: [PATCH 1/2] first patch\n' + b'Message-ID: \n' + b'\n' + b'First patch body.\n' + b'\n' + b'From nobody Thu Jan 1 00:00:00 1970\n' + b'From: Test \n' + b'Subject: [PATCH 2/2] second patch\n' + b'Message-ID: \n' + b'\n' + b'Second patch body.\n' + ) + + def test_parse_messages(self): + mbox = bytes_to_mbox(self.MBOX_DATA) + messages = list(mbox) + self.assertEqual(len(messages), 2) + self.assertIn('patch1@example.com', messages[0].get('Message-ID')) + self.assertIn('patch2@example.com', messages[1].get('Message-ID')) + + def test_get_bytes_preserves_content(self): + mbox = bytes_to_mbox(self.MBOX_DATA) + for key, msg in mbox.iteritems(): + raw = mbox.get_bytes(key) + self.assertIn(b'From: Test ', raw) + self.assertIn(msg.get('Subject').encode(), raw) + + def test_empty_input(self): + mbox = bytes_to_mbox(b'') + self.assertEqual(len(list(mbox)), 0) + + +class IngestEmailsTest(TestCase): + def _make_mbox(self, messages): + buf = b'' + for msg in messages: + buf += b'From nobody Thu Jan 1 00:00:00 1970\n' + buf += msg + b'\n' + return bytes_to_mbox(buf) + + def test_ingest_creates_metadata(self): + project = create_project() + series = create_series(project=project) + patch = create_patches(count=1, series=series)[0] + + mbox = self._make_mbox( + [ + b'From: Test \n' + b'Subject: [PATCH] fix thing\n' + b'Message-ID: \n' + b'\nBody\n', + ] + ) + + backend = MagicMock() + backend.series_metadata.return_value = { + 'github_pr': 'https://github.com/o/r/pull/1', + 'github_branch': 'fix-thing', + } + forge_config = MagicMock() + forge_config.project = project + + with mock_patch('patchwork.forge.util.parse_mail', return_value=patch): + ingest_emails(mbox, backend, forge_config, ForgeEvent()) + + backend.series_metadata.assert_called_once() + self.assertTrue( + SeriesMetadata.objects.filter( + series=series, key='github_pr' + ).exists() + ) + self.assertTrue( + SeriesMetadata.objects.filter( + series=series, key='github_branch' + ).exists() + ) + + def test_ingest_skips_duplicates(self): + from patchwork.parser import DuplicateMailError + + mbox = self._make_mbox( + [ + b'From: Test \n' + b'Subject: [PATCH] fix thing\n' + b'Message-ID: \n' + b'\nBody\n', + ] + ) + + backend = MagicMock() + forge_config = MagicMock() + forge_config.project = create_project() + + with mock_patch( + 'patchwork.forge.util.parse_mail', + side_effect=DuplicateMailError(msgid=''), + ): + ingest_emails(mbox, backend, forge_config, ForgeEvent()) + + backend.series_metadata.assert_not_called() + + def test_ingest_skips_value_errors(self): + mbox = self._make_mbox( + [ + b'From: Test \n' + b'Subject: [PATCH] fix thing\n' + b'Message-ID: \n' + b'\nBody\n', + ] + ) + + backend = MagicMock() + forge_config = MagicMock() + forge_config.project = create_project() + + with mock_patch( + 'patchwork.forge.util.parse_mail', + side_effect=ValueError('bad email'), + ): + ingest_emails(mbox, backend, forge_config, ForgeEvent()) + + backend.series_metadata.assert_not_called() + + def test_ingest_no_metadata_on_empty_values(self): + project = create_project() + series = create_series(project=project) + patch = create_patches(count=1, series=series)[0] + + mbox = self._make_mbox( + [ + b'From: Test \n' + b'Subject: [PATCH] fix thing\n' + b'Message-ID: \n' + b'\nBody\n', + ] + ) + + backend = MagicMock() + backend.series_metadata.return_value = { + 'github_pr': 'https://github.com/o/r/pull/1', + 'github_branch': '', + } + forge_config = MagicMock() + forge_config.project = project + + with mock_patch('patchwork.forge.util.parse_mail', return_value=patch): + ingest_emails(mbox, backend, forge_config, ForgeEvent()) + + self.assertTrue( + SeriesMetadata.objects.filter( + series=series, key='github_pr' + ).exists() + ) + self.assertFalse( + SeriesMetadata.objects.filter( + series=series, key='github_branch' + ).exists() + ) + + +class SendEmailsTest(TestCase): + MBOX_DATA = ( + b'From nobody Thu Jan 1 00:00:00 1970\n' + b'From: Author \n' + b'Sender: Bot \n' + b'To: list@example.com\n' + b'Cc: reviewer@example.com\n' + b'Subject: [PATCH 1/1] fix thing\n' + b'Message-ID: \n' + b'\n' + b'Patch body.\n' + ) + + def test_sends_via_smtp(self): + mbox = bytes_to_mbox(self.MBOX_DATA) + forge_config = MagicMock() + + mock_conn = MagicMock() + mock_conn.connection.sendmail.return_value = {} + + with mock_patch( + 'patchwork.forge.util.get_connection' + ) as mock_get_conn: + mock_get_conn.return_value.__enter__ = MagicMock( + return_value=mock_conn + ) + mock_get_conn.return_value.__exit__ = MagicMock(return_value=False) + send_emails(mbox, forge_config) + + mock_conn.connection.sendmail.assert_called_once() + call_args = mock_conn.connection.sendmail.call_args + sender = call_args[0][0] + recipients = call_args[0][1] + raw_bytes = call_args[0][2] + self.assertEqual(sender, 'bot@example.com') + self.assertIn('author@example.com', recipients) + self.assertIn('list@example.com', recipients) + self.assertIn('reviewer@example.com', recipients) + self.assertIn(b'[PATCH 1/1] fix thing', raw_bytes) + + def test_sends_raw_bytes(self): + mbox = bytes_to_mbox(self.MBOX_DATA) + forge_config = MagicMock() + + mock_conn = MagicMock() + mock_conn.connection.sendmail.return_value = {} + + with mock_patch( + 'patchwork.forge.util.get_connection' + ) as mock_get_conn: + mock_get_conn.return_value.__enter__ = MagicMock( + return_value=mock_conn + ) + mock_get_conn.return_value.__exit__ = MagicMock(return_value=False) + send_emails(mbox, forge_config) + + raw_bytes = mock_conn.connection.sendmail.call_args[0][2] + self.assertIn(b'Sender: Bot ', raw_bytes) + self.assertIn(b'Message-ID: ', raw_bytes) From patchwork Thu Jun 4 14:18:17 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 0077C1BC4353 for ; Thu, 04 Jun 2026 16:18:29 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 08/16] forge: add git mirror utilities Date: Thu, 4 Jun 2026 16:18:17 +0200 Message-ID: <20260604141826.2998337-9-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 105 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add a GitMirror class that manages bare git clones for forge sync operations. It handles mirror creation, fetching with temporary credential files, worktree lifecycle, and patch generation via git format-patch with support for cover letters, version numbering, range-diff, custom headers and threading. All git commands run with isolated configuration to prevent interference from user or system git settings. Signed-off-by: Robin Jarry --- patchwork/forge/__init__.py | 8 + patchwork/forge/git.py | 274 +++++++++++++++++++++++++++++ patchwork/settings/base.py | 3 + patchwork/tests/forge/test_git.py | 278 ++++++++++++++++++++++++++++++ 4 files changed, 563 insertions(+) create mode 100644 patchwork/forge/git.py create mode 100644 patchwork/tests/forge/test_git.py diff --git a/patchwork/forge/__init__.py b/patchwork/forge/__init__.py index 1c236e4a97f2..d9ac7486d609 100644 --- a/patchwork/forge/__init__.py +++ b/patchwork/forge/__init__.py @@ -158,6 +158,14 @@ class ForgeBackend(ABC): auth.update(repo_overrides) return auth + def git_credentials(self, forge_config): + """ + Return git credential store content as a string for the given + project. Written to a temporary file and passed to git via + GIT_CREDENTIAL_HELPER during clone and fetch operations. + """ + raise NotImplementedError + _backends = {} diff --git a/patchwork/forge/git.py b/patchwork/forge/git.py new file mode 100644 index 000000000000..311c04288ba9 --- /dev/null +++ b/patchwork/forge/git.py @@ -0,0 +1,274 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +""" +Git mirror management for forge sync operations. +""" + +import contextlib +import email +import logging +import os +import re +import subprocess +import tempfile + +from django.conf import settings + +from patchwork.forge.util import bytes_to_mbox +from patchwork.forge.util import sender_identity + +logger = logging.getLogger(__name__) + + +class GitMirror: + """ + Bare mirror clone of a forge repository. + + Provides worktree-based operations for patch generation. All git commands + run with isolated configuration (no user/system gitconfig, no interactive + prompts) and temporary credential files that are removed after each + operation. + """ + + def __init__(self, backend, forge_config): + self.mirror_path = os.path.join( + settings.FORGE_GIT_MIRROR_PATH, + f'{forge_config.project.linkname}.git', + ) + self.backend = backend + self.forge_config = forge_config + self.repo_url = backend.repo_url(forge_config) + self.auth = backend.get_auth(forge_config) + self.__worktree = None + self.__credentials = None + + def repo_dir(self): + if self.__worktree: + return self.__worktree + return self.mirror_path + + def git(self, *args, **kwargs): + env = dict(os.environ) + env.update( + { + 'GIT_CONFIG_GLOBAL': '/dev/null', + 'GIT_CONFIG_SYSTEM': '/dev/null', + 'GIT_TERMINAL_PROMPT': '0', + } + ) + cmd = ['git'] + if os.path.isdir(self.repo_dir()): + cmd += ['-C', self.repo_dir()] + if self.__credentials: + cmd += [ + '-c', + f'credential.helper=store --file={self.__credentials}', + ] + cmd.extend(args) + logger.debug('+ %s', ' '.join(cmd)) + return subprocess.run(cmd, env=env, text=False, check=True, **kwargs) + + def git_output(self, *args, **kwargs): + result = self.git(*args, capture_output=True, **kwargs) + return result.stdout.decode('utf-8', errors='surrogateescape').strip() + + @contextlib.contextmanager + def credentials(self): + with tempfile.NamedTemporaryFile( + prefix='patchwork-cred-', mode='w', delete_on_close=False + ) as tmp: + tmp.write(self.backend.git_credentials(self.forge_config)) + tmp.close() + try: + self.__credentials = tmp.name + yield + finally: + self.__credentials = None + + def ensure_mirror(self, fetch_refs=None): + """ + Create the bare mirror clone if it does not exist yet and configure the + refspec to also fetch pull request heads. + """ + head_path = os.path.join(self.mirror_path, 'HEAD') + if not os.path.exists(head_path): + logger.info('cloning mirror to %s', self.mirror_path) + os.makedirs(os.path.dirname(self.mirror_path), exist_ok=True) + with self.credentials(): + self.git('clone', '--mirror', self.repo_url, self.mirror_path) + + if fetch_refs: + self.git( + 'config', '--replace-all', 'remote.origin.fetch', fetch_refs + ) + + def fetch(self): + """ + Fetch all remotes and prune stale references. + """ + logger.info('fetching mirror %s', self.mirror_path) + with self.credentials(): + self.git('fetch', '--all', '--prune') + + def add_worktree(self, ref, path): + """ + Create a temporary worktree checked out at the given ref. + """ + self.git('worktree', 'add', '-fd', '--checkout', path, ref) + + def del_worktree(self, path): + """ + Remove a previously created worktree. + """ + self.git('worktree', 'remove', '-ff', path) + + @contextlib.contextmanager + def worktree(self, ref): + w = tempfile.mkdtemp(prefix='patchwork-worktree-') + try: + self.add_worktree(ref, w) + self.__worktree = w + yield + finally: + self.__worktree = None + self.del_worktree(w) + + def commit_count(self, base_ref): + """ + Return the number of commits in base_ref..HEAD. + """ + out = self.git_output('rev-list', '--count', f'{base_ref}..HEAD') + return int(out) + + def ref_exists(self, ref): + """ + Return True if ref exists in the repository. + """ + try: + self.git_output('cat-file', '-t', ref) + return True + except subprocess.CalledProcessError: + return False + + RECIPIENT_RE = re.compile(r'\s*\d+\s+(?P.+)\s+<(?P.+@.+)>') + + def recipients(self, base_ref): + out = self.git_output( + 'shortlog', + '-se', + '-w0', + '--group=author', + '--group=committer', + '--group=trailer:cc', + '--group=trailer:acked-by', + '--group=trailer:co-authored-by', + '--group=trailer:reported-by', + '--group=trailer:requested-by', + '--group=trailer:reviewed-by', + '--group=trailer:signed-off-by', + '--group=trailer:suggested-by', + '--group=trailer:tested-by', + f'{base_ref}..HEAD', + ) + recipients = {} + for m in self.RECIPIENT_RE.finditer(out): + name = m.group('name') + name = re.sub(r'\w\w+', lambda s: s.group(0).title(), name) + name = name.strip('"\' \t') + addr = m.group('email').lower() + recipients[addr] = name + for addr, name in recipients.items(): + yield email.utils.formataddr((name, addr)) + + def add_commit_notes(self, base_ref, note_fn): + """ + Add a git note to each commit in base_ref..HEAD. + + note_fn(sha) is called for each commit and should return the + note text, or None to skip. + """ + out = self.git_output('rev-list', f'{base_ref}..HEAD') + for sha in out.splitlines(): + sha = sha.strip() + if not sha: + continue + note = note_fn(sha) + if note: + self.git('notes', 'add', '-f', '-m', note, sha) + + def format_patches( + self, + base_ref, + user, + version=1, + cover_title=None, + cover_body=None, + range_diff_base=None, + in_reply_to=None, + ): + """ + Generate patches for commits in base_ref..HEAD. + + When the series has more than one commit and a cover_title is provided, + a cover letter is generated. For respins (version > 1), --in-reply-to + threads the cover letter under the original and --range-diff shows what + changed since the previous version. + + Returns mailbox.mbox object containing all messages. + """ + name, addr = sender_identity(user, self.forge_config) + args = [ + '-c', + f'user.name={name}', + '-c', + f'user.email={addr}', + 'format-patch', + '--stdout', + '--notes', + '--thread=shallow', + f'--subject-prefix=PATCH {self.forge_config.project.linkname}', + f'--to={self.forge_config.project.listemail}', + ] + + for cc in self.recipients(base_ref): + args.append(f'--cc={cc}') + + extra_headers = { + 'Sender': self.forge_config.sender_email, + 'Reply-To': self.forge_config.project.listemail, + 'List-ID': f'<{self.forge_config.project.listid}>', + 'X-Patchwork-Hint': 'ignore', + } + for key, value in extra_headers.items(): + args.append(f'--add-header={key}: {value}') + + if in_reply_to: + args.append(f'--in-reply-to={in_reply_to}') + + if version > 1: + args.append(f'-v{version}') + + if self.commit_count(base_ref) > 1 and cover_title: + args.append('--cover-letter') + if cover_body: + desc = f'{cover_title}\n\n{cover_body}' + else: + desc = cover_title + desc_file = os.path.join(self.repo_dir(), '.cover-description') + with open(desc_file, 'w') as f: + f.write(desc) + args += [ + '--cover-from-description=subject', + f'--description-file={desc_file}', + ] + + if range_diff_base and self.ref_exists(range_diff_base): + args.append(f'--range-diff={base_ref}..{range_diff_base}') + + args.append(f'{base_ref}..HEAD') + + result = self.git(*args, capture_output=True) + return bytes_to_mbox(result.stdout) diff --git a/patchwork/settings/base.py b/patchwork/settings/base.py index e5bcbacf05dc..bc42b14b2242 100644 --- a/patchwork/settings/base.py +++ b/patchwork/settings/base.py @@ -303,3 +303,6 @@ FORGE_WEBHOOK_SECRETS = {} # }, # } FORGE_AUTH = {} + +# Base directory for git mirror clones (one bare repo per project) +FORGE_GIT_MIRROR_PATH = '' diff --git a/patchwork/tests/forge/test_git.py b/patchwork/tests/forge/test_git.py new file mode 100644 index 000000000000..64e187c89b20 --- /dev/null +++ b/patchwork/tests/forge/test_git.py @@ -0,0 +1,278 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import os +import shutil +import subprocess +import tempfile +import unittest + +from django.test import TestCase +from django.test import override_settings + +from patchwork.forge import ForgeUser +from patchwork.forge.git import GitMirror + + +def _has_git(): + try: + subprocess.run( + ['git', '--version'], + capture_output=True, + check=True, + ) + return True + except (FileNotFoundError, subprocess.CalledProcessError): + return False + + +def _run_git(*args, cwd=None): + env = dict(os.environ) + env.update( + { + 'GIT_CONFIG_GLOBAL': '/dev/null', + 'GIT_CONFIG_SYSTEM': '/dev/null', + 'GIT_TERMINAL_PROMPT': '0', + 'GIT_AUTHOR_NAME': 'Test Author', + 'GIT_AUTHOR_EMAIL': 'author@example.com', + 'GIT_COMMITTER_NAME': 'Test Author', + 'GIT_COMMITTER_EMAIL': 'author@example.com', + } + ) + return subprocess.run( + ['git'] + list(args), + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=True, + ) + + +@unittest.skipUnless(_has_git(), 'git is not installed') +class GitMirrorTestBase(TestCase): + """ + Base class that creates a bare "upstream" repo with a few commits + and a GitMirror clone of it. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.tmpdir = tempfile.mkdtemp(prefix='patchwork-test-git-') + + # create upstream bare repo + cls.upstream_path = os.path.join(cls.tmpdir, 'upstream.git') + work = os.path.join(cls.tmpdir, 'work') + os.makedirs(work) + _run_git('init', cwd=work) + _run_git('commit', '--allow-empty', '-m', 'initial', cwd=work) + + # tag the base for commit range + _run_git('tag', 'base', cwd=work) + + # add commits + with open(os.path.join(work, 'a.txt'), 'w') as f: + f.write('aaa\n') + _run_git('add', 'a.txt', cwd=work) + _run_git( + 'commit', + '-m', + 'add file a\n\nSigned-off-by: Test Author ', + cwd=work, + ) + + with open(os.path.join(work, 'b.txt'), 'w') as f: + f.write('bbb\n') + _run_git('add', 'b.txt', cwd=work) + _run_git( + 'commit', + '-m', + 'add file b\n\nAcked-by: Reviewer ', + cwd=work, + ) + + # clone as bare + _run_git('clone', '--mirror', work, cls.upstream_path) + shutil.rmtree(work) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmpdir, ignore_errors=True) + super().tearDownClass() + + def _make_mirror(self): + from unittest.mock import MagicMock + + backend = MagicMock() + backend.repo_url.return_value = self.upstream_path + backend.get_auth.return_value = {} + backend.git_credentials.return_value = '' + + forge_config = MagicMock() + forge_config.project.linkname = 'mirror' + forge_config.project.listemail = 'list@example.com' + forge_config.project.listid = 'list.example.com' + forge_config.sender_email = 'patchwork@example.com' + + with override_settings(FORGE_GIT_MIRROR_PATH=self.tmpdir): + mirror = GitMirror(backend, forge_config) + + return mirror + + +class CommitCountTest(GitMirrorTestBase): + def test_count_commits(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + count = mirror.commit_count('base') + self.assertEqual(count, 2) + + +class RefExistsTest(GitMirrorTestBase): + def test_existing_ref(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + self.assertTrue(mirror.ref_exists('base')) + + def test_missing_ref(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + self.assertFalse(mirror.ref_exists('nonexistent')) + + +class TempWorktreeTest(GitMirrorTestBase): + def test_creates_and_cleans_up(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + w = None + with mirror.worktree('HEAD'): + w = mirror.repo_dir() + self.assertTrue(os.path.isdir(w)) + self.assertTrue(os.path.exists(os.path.join(w, 'a.txt'))) + self.assertTrue(os.path.exists(os.path.join(w, 'b.txt'))) + self.assertFalse(os.path.isdir(w)) + + +class RecipientsTest(GitMirrorTestBase): + def test_extracts_recipients(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + recipients = list(mirror.recipients('base')) + self.assertIn('Test Author ', recipients) + self.assertIn('Reviewer ', recipients) + + +class FormatPatchesTest(GitMirrorTestBase): + def test_single_patch(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + user = ForgeUser( + login='author', name='Test Author', email='author@example.com' + ) + mbox = mirror.format_patches('HEAD~1', user) + messages = list(mbox) + self.assertEqual(len(messages), 1) + self.assertIn('[PATCH', messages[0].get('Subject')) + + def test_multi_patch_with_cover(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + user = ForgeUser( + login='author', name='Test Author', email='author@example.com' + ) + mbox = mirror.format_patches( + 'base', + user, + cover_title='Test series', + cover_body='This is a test.', + ) + messages = list(mbox) + # cover letter + 2 patches + self.assertEqual(len(messages), 3) + subjects = [m.get('Subject') for m in messages] + self.assertTrue( + any('0/2' in s for s in subjects), + f'no cover letter: {subjects}', + ) + + def test_version_numbering(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + user = ForgeUser( + login='author', name='Test Author', email='author@example.com' + ) + mbox = mirror.format_patches('base', user, version=2) + messages = list(mbox) + subjects = [m.get('Subject') for m in messages] + self.assertTrue( + all('v2' in s for s in subjects), f'no v2: {subjects}' + ) + + def test_extra_headers(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + user = ForgeUser( + login='author', name='Test Author', email='author@example.com' + ) + mbox = mirror.format_patches('HEAD~1', user) + messages = list(mbox) + msg = messages[0] + self.assertIn('ignore', msg.get('X-Patchwork-Hint', '')) + self.assertIn('list.example.com', msg.get('List-ID', '')) + self.assertIn('list@example.com', msg.get('Reply-To', '')) + self.assertIn('patchwork@example.com', msg.get('Sender', '')) + + def test_in_reply_to(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + user = ForgeUser( + login='author', name='Test Author', email='author@example.com' + ) + mbox = mirror.format_patches( + 'base', + user, + cover_title='Test', + in_reply_to='', + ) + messages = list(mbox) + cover = messages[0] + self.assertIn( + 'v1-cover@example.com', + cover.get('In-Reply-To', ''), + ) + + def test_cc_from_trailers(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + user = ForgeUser( + login='author', name='Test Author', email='author@example.com' + ) + mbox = mirror.format_patches('base', user) + messages = list(mbox) + # check that reviewer from Acked-by is in Cc + all_cc = ' '.join(m.get('Cc', '') for m in messages) + self.assertIn('reviewer@example.com', all_cc) + + def test_to_header(self): + mirror = self._make_mirror() + mirror.ensure_mirror() + with mirror.worktree('HEAD'): + user = ForgeUser( + login='author', name='Test Author', email='author@example.com' + ) + mbox = mirror.format_patches('HEAD~1', user) + messages = list(mbox) + self.assertIn('list@example.com', messages[0].get('To', '')) From patchwork Thu Jun 4 14:18:18 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 1709F1BC4354 for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 09/16] forge: sync github pull requests to mailing list Date: Thu, 4 Jun 2026 16:18:18 +0200 Message-ID: <20260604141826.2998337-10-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 106 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add the GitHub forge backend with webhook signature verification, pull request event parsing, and forge-to-mailing-list sync. When a pull request is opened or force-pushed, fetch the PR head into a local git mirror, generate patches via git format-patch, ingest them into the database, store series metadata linking back to the PR, and send the patches to the mailing list. Respin detection increments the version number and optionally threads the new series under the original when thread_respins is enabled. Signed-off-by: Robin Jarry --- patchwork/forge/github/__init__.py | 83 ++++++++++++++++++++++++++++++ patchwork/forge/github/to_ml.py | 54 +++++++++++++++++++ patchwork/forge/github/webhook.py | 43 ++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 patchwork/forge/github/__init__.py create mode 100644 patchwork/forge/github/to_ml.py create mode 100644 patchwork/forge/github/webhook.py diff --git a/patchwork/forge/github/__init__.py b/patchwork/forge/github/__init__.py new file mode 100644 index 000000000000..ded7402cd5f3 --- /dev/null +++ b/patchwork/forge/github/__init__.py @@ -0,0 +1,83 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +""" +GitHub forge backend. +""" + +import hashlib +import hmac +import json +import logging + +from patchwork.forge import ForgeBackend +from patchwork.forge import register_backend +from patchwork.forge.github.to_ml import handle_pull_request +from patchwork.forge.github.webhook import parse_pull_request + +logger = logging.getLogger(__name__) + + +class GitHubBackend(ForgeBackend): + def verify_webhook_signature(self, body, headers, secret): + if not secret: + return True + signature = headers.get('X-Hub-Signature-256', '') + prefix = 'sha256=' + if not signature.startswith(prefix): + return False + try: + sig = bytes.fromhex(signature[len(prefix) :]) + except ValueError: + return False + mac = hmac.new(secret.encode(), body, hashlib.sha256) + return hmac.compare_digest(sig, mac.digest()) + + def parse_webhook_event(self, body, headers): + event_type = headers.get('X-GitHub-Event', '') + payload = json.loads(body) + + parsers = { + 'pull_request': parse_pull_request, + } + + parser = parsers.get(event_type) + if parser is None: + return None + return parser(payload) + + def repo_url(self, forge_config): + return f'https://github.com/{forge_config.repo}.git' + + def pr_ref(self, forge_config, pr_number): + return f'https://github.com/{forge_config.repo}/pull/{pr_number}' + + def pr_refspec(self, pr_number): + return f'pull/{pr_number}/head' + + def meta_key_pr(self): + return 'github_pr' + + def series_metadata(self, forge_config, event): + return { + 'github_pr': self.pr_ref(forge_config, event.pr_number), + 'github_branch': event.pr_head_branch, + } + + def git_credentials(self, forge_config): + auth = self.get_auth(forge_config) + token = auth.get('token', '') + return f'https://x-access-token:{token}@github.com\n' + + def process_webhook_event(self, forge_config, event): + handlers = { + 'pull_request': handle_pull_request, + } + handler = handlers.get(event.type) + if handler: + handler(self, forge_config, event) + + +register_backend('github', GitHubBackend()) diff --git a/patchwork/forge/github/to_ml.py b/patchwork/forge/github/to_ml.py new file mode 100644 index 000000000000..fac07b180a51 --- /dev/null +++ b/patchwork/forge/github/to_ml.py @@ -0,0 +1,54 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +from patchwork.forge.git import GitMirror +from patchwork.forge.util import ingest_emails +from patchwork.forge.util import next_version +from patchwork.forge.util import sanitize_pr_body +from patchwork.forge.util import send_emails + + +def handle_pull_request(gh, forge_config, event): + if not forge_config.sync_forge_to_ml: + return + + mirror = GitMirror(gh, forge_config) + mirror.ensure_mirror() + mirror.fetch() + + version = 1 + in_reply_to = '' + range_diff_base = '' + if event.pr_action == 'synchronize': + version, reply_msgid, range_diff_base = next_version( + gh, forge_config, event + ) + if forge_config.thread_respins: + in_reply_to = reply_msgid + + pr_url = gh.pr_ref(forge_config, event.pr_number) + cover_body = sanitize_pr_body(event.pr_body) + if cover_body: + cover_body += f'\n\nPull request: {pr_url}' + else: + cover_body = f'Pull request: {pr_url}' + + with mirror.worktree(event.pr_head): + mirror.add_commit_notes( + event.pr_base, + lambda sha: f'{pr_url}/commits/{sha}', + ) + mbox = mirror.format_patches( + event.pr_base, + event.author, + version=version, + cover_title=event.pr_title, + cover_body=cover_body, + range_diff_base=range_diff_base, + in_reply_to=in_reply_to, + ) + + ingest_emails(mbox, gh, forge_config, event) + send_emails(mbox, forge_config) diff --git a/patchwork/forge/github/webhook.py b/patchwork/forge/github/webhook.py new file mode 100644 index 000000000000..3945a6ff1ef4 --- /dev/null +++ b/patchwork/forge/github/webhook.py @@ -0,0 +1,43 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + +from patchwork.forge import ForgeEvent +from patchwork.forge import ForgeUser + + +def parse_pull_request(payload): + action = payload.get('action', '') + if action not in ('opened', 'synchronize'): + return None + pr = payload.get('pull_request', {}) + pr_body = pr.get('body') or '' + return ForgeEvent( + type='pull_request', + repo_key=get_repo_key(payload), + pr_number=pr.get('number', 0), + author=parse_user(pr.get('user')), + pr_title=pr.get('title', ''), + pr_body=pr_body, + pr_head=f'pull/{pr.get("number", 0)}/head', + pr_base=pr.get('base', {}).get('sha', ''), + pr_head_branch=pr.get('head', {}).get('ref', ''), + pr_action=action, + pr_before=payload.get('before', ''), + ) + + +def get_repo_key(payload): + repo = payload.get('repository', {}) + return repo.get('full_name', '').lower() + + +def parse_user(user): + if not user: + return ForgeUser() + return ForgeUser( + login=user.get('login', ''), + name=user.get('name', ''), + email=user.get('email', ''), + ) From patchwork Thu Jun 4 14:18:19 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 2B38E1BC4355 for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 10/16] forge: sync github comments and reviews to mailing list Date: Thu, 4 Jun 2026 16:18:19 +0200 Message-ID: <20260604141826.2998337-11-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 107 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Parse issue_comment and pull_request_review webhook events and forward them to the mailing list as email replies to the corresponding patch series. Replies are built as proper RFC 2822 messages with From set to the forge user, Sender to the patchwork bot, and In-Reply-To/References pointing to the series cover letter or first patch. A X-Patchwork-Hint: ignore header prevents parsemail from re-ingesting the emails on their way back. Each reply is ingested directly into the database as a comment before being sent to the list, following the same ingest-then-send approach used for patches. Reviews include the review state, body text, and inline comments with file paths and quoted diff context. Signed-off-by: Robin Jarry --- patchwork/forge/github/__init__.py | 8 +++ patchwork/forge/github/api.py | 66 ++++++++++++++++++++++ patchwork/forge/github/to_ml.py | 91 ++++++++++++++++++++++++++++++ patchwork/forge/github/webhook.py | 34 +++++++++++ 4 files changed, 199 insertions(+) create mode 100644 patchwork/forge/github/api.py diff --git a/patchwork/forge/github/__init__.py b/patchwork/forge/github/__init__.py index ded7402cd5f3..4441c2b11ab0 100644 --- a/patchwork/forge/github/__init__.py +++ b/patchwork/forge/github/__init__.py @@ -14,8 +14,12 @@ import logging from patchwork.forge import ForgeBackend from patchwork.forge import register_backend +from patchwork.forge.github.to_ml import handle_issue_comment from patchwork.forge.github.to_ml import handle_pull_request +from patchwork.forge.github.to_ml import handle_review +from patchwork.forge.github.webhook import parse_issue_comment from patchwork.forge.github.webhook import parse_pull_request +from patchwork.forge.github.webhook import parse_review logger = logging.getLogger(__name__) @@ -40,7 +44,9 @@ class GitHubBackend(ForgeBackend): payload = json.loads(body) parsers = { + 'issue_comment': parse_issue_comment, 'pull_request': parse_pull_request, + 'pull_request_review': parse_review, } parser = parsers.get(event_type) @@ -73,7 +79,9 @@ class GitHubBackend(ForgeBackend): def process_webhook_event(self, forge_config, event): handlers = { + 'issue_comment': handle_issue_comment, 'pull_request': handle_pull_request, + 'review': handle_review, } handler = handlers.get(event.type) if handler: diff --git a/patchwork/forge/github/api.py b/patchwork/forge/github/api.py new file mode 100644 index 000000000000..6e99592a9eac --- /dev/null +++ b/patchwork/forge/github/api.py @@ -0,0 +1,66 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + + +import json +import logging +import urllib.error +import urllib.request + +from patchwork.forge import ReviewComment + +logger = logging.getLogger(__name__) + + +class GitHubAPIError(Exception): + def __init__(self, method, path, code, body): + super().__init__(f'API {method} {path} failed ({code}): {body}') + + +def gh_api_request(gh, forge_config, method, path, data=None): + """ + Make a GitHub API request. Return the parsed JSON response. + """ + auth = gh.get_auth(forge_config) + token = auth.get('token', '') + url = f'https://api.github.com{path}' + body = json.dumps(data).encode() if data else None + req = urllib.request.Request(url, data=body, method=method) + req.add_header('Authorization', f'token {token}') + req.add_header('Accept', 'application/vnd.github+json') + if body: + req.add_header('Content-Type', 'application/json') + try: + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + error_body = e.read().decode('utf-8', errors='replace') + raise GitHubAPIError(method, path, e.code, error_body) from e + + +def fetch_review_comments(gh, forge_config, pr_number, review_id): + """ + Fetch inline comments for a review via the GitHub API. + """ + owner, repo = forge_config.repo.split('/', 1) + results = gh_api_request( + gh, + forge_config, + 'GET', + f'/repos/{owner}/{repo}/pulls/{pr_number}/reviews/{review_id}/comments', + ) + comments = [] + for r in results: + body = r.get('body') or '' + if body == '': + continue + comments.append( + ReviewComment( + body=body, + path=r.get('path', '/dev/null'), + diff_hunk=r.get('diff_hunk', ''), + ) + ) + return comments diff --git a/patchwork/forge/github/to_ml.py b/patchwork/forge/github/to_ml.py index fac07b180a51..55ee03a9ebe0 100644 --- a/patchwork/forge/github/to_ml.py +++ b/patchwork/forge/github/to_ml.py @@ -3,11 +3,26 @@ # # SPDX-License-Identifier: GPL-2.0-or-later +import email +import logging + +from django.contrib.auth import get_user_model +from django.db import transaction +from django.utils.text import slugify + +from patchwork.forge import ReviewComment from patchwork.forge.git import GitMirror +from patchwork.forge.github.api import fetch_review_comments +from patchwork.forge.util import bytes_to_mbox +from patchwork.forge.util import find_series_by_pr from patchwork.forge.util import ingest_emails from patchwork.forge.util import next_version +from patchwork.forge.util import reply_to_msgid from patchwork.forge.util import sanitize_pr_body from patchwork.forge.util import send_emails +from patchwork.forge.util import sender_identity + +logger = logging.getLogger(__name__) def handle_pull_request(gh, forge_config, event): @@ -52,3 +67,79 @@ def handle_pull_request(gh, forge_config, event): ingest_emails(mbox, gh, forge_config, event) send_emails(mbox, forge_config) + + +def handle_issue_comment(gh, forge_config, event): + series = find_series_by_pr(gh, forge_config, event.pr_number).last() + if not series: + logger.warning('no series found for PR #%d', event.pr_number) + return + + subject = f'Re: {series.name} (comment)' + reply(gh, forge_config, event, series, subject, event.body) + + +def handle_review(gh, forge_config, event): + series = find_series_by_pr(gh, forge_config, event.pr_number).last() + if not series: + logger.warning('no series found for PR #%d', event.pr_number) + return + + subject = f'Re: {series.name} (review: {event.review_state})' + + body = '' + if event.review_state: + body = f'Review: {event.review_state}\n\n' + if event.body: + body += f'{event.body}\n\n' + + comments = fetch_review_comments( + gh, forge_config, event.pr_number, event.review_id + ) + for c in comments: + body += f'--- {c.path}\n' + if c.diff_hunk: + for line in c.diff_hunk.split('\n'): + body += f'> {line}\n' + body += '\n' + body += f'{c.body}\n\n' + + if not event.body and not comments: + return + + reply(gh, forge_config, event, series, subject, body.rstrip()) + + +def reply(gh, forge_config, event, series, subject, body): + """ + Build a reply email as an mbox, ingest it into the database and + send it to the mailing list. + """ + in_reply_to = reply_to_msgid(series) + if not in_reply_to: + logger.warning('no message-id for series %d', series.id) + return + + _, addr = email.utils.parseaddr(forge_config.sender_addr) + uid, domain = addr.rsplit('@', 1) + + msg = email.mime.text.MIMEText(body) + msg['From'] = email.utils.formataddr( + sender_identity(event.author, forge_config) + ) + msg['Sender'] = email.utils.formataddr( + email.utils.parseaddr(forge_config.sender_addr) + ) + msg['To'] = forge_config.project.listemail + msg['Subject'] = email.header.Header(subject, 'utf-8') + msg['Date'] = email.utils.formatdate(localtime=True) + msg['Message-ID'] = email.utils.make_msgid(uid, domain) + msg['In-Reply-To'] = in_reply_to + msg['References'] = in_reply_to + msg['Reply-To'] = forge_config.project.listemail + msg['List-ID'] = f'<{forge_config.project.listid}>' + msg['X-Patchwork-Hint'] = 'ignore' + + mbox = bytes_to_mbox(msg.as_bytes(unixfrom=True)) + ingest_emails(mbox, gh, forge_config, event) + send_emails(mbox, forge_config) diff --git a/patchwork/forge/github/webhook.py b/patchwork/forge/github/webhook.py index 3945a6ff1ef4..bc788512265a 100644 --- a/patchwork/forge/github/webhook.py +++ b/patchwork/forge/github/webhook.py @@ -41,3 +41,37 @@ def parse_user(user): name=user.get('name', ''), email=user.get('email', ''), ) + + +def parse_issue_comment(payload): + if payload.get('action') != 'created': + return None + issue = payload.get('issue', {}) + if 'pull_request' not in issue: + return None + comment = payload.get('comment', {}) + comment_body = comment.get('body') or '' + return ForgeEvent( + type='issue_comment', + repo_key=get_repo_key(payload), + pr_number=issue.get('number', 0), + author=parse_user(comment.get('user')), + body=comment_body, + ) + + +def parse_review(payload): + if payload.get('action') != 'submitted': + return None + review = payload.get('review', {}) + pr = payload.get('pull_request', {}) + review_body = review.get('body') or '' + return ForgeEvent( + type='review', + repo_key=get_repo_key(payload), + pr_number=pr.get('number', 0), + review_id=review.get('id', 0), + author=parse_user(review.get('user')), + body=review_body, + review_state=review.get('state', ''), + ) From patchwork Thu Jun 4 14:18:20 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 408F11BC4356 for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 11/16] fixup! forge: sync github comments and reviews to mailing list Date: Thu, 4 Jun 2026 16:18:20 +0200 Message-ID: <20260604141826.2998337-12-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 108 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit --- patchwork/forge/github/to_ml.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/patchwork/forge/github/to_ml.py b/patchwork/forge/github/to_ml.py index 55ee03a9ebe0..27797094cbe7 100644 --- a/patchwork/forge/github/to_ml.py +++ b/patchwork/forge/github/to_ml.py @@ -6,11 +6,6 @@ import email import logging -from django.contrib.auth import get_user_model -from django.db import transaction -from django.utils.text import slugify - -from patchwork.forge import ReviewComment from patchwork.forge.git import GitMirror from patchwork.forge.github.api import fetch_review_comments from patchwork.forge.util import bytes_to_mbox From patchwork Thu Jun 4 14:18:21 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 612B21BC4357 for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 12/16] forge: sync github check results to patchwork Date: Thu, 4 Jun 2026 16:18:21 +0200 Message-ID: <20260604141826.2998337-13-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 109 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Parse check_run and check_suite webhook events. When a check run is created, record a pending Check on each patch of the linked series. When a check suite completes, record the final state (success, fail or warning) for each individual check run. A summary email listing all check run names, statuses and URLs is sent to the mailing list as a reply to the series, following the same ingest-then-send pipeline as comments and reviews. Checks are created under a dedicated "github" system user and use the check run name as context slug. Signed-off-by: Robin Jarry --- patchwork/forge/github/__init__.py | 8 +++ patchwork/forge/github/api.py | 30 +++++++++++ patchwork/forge/github/to_ml.py | 86 ++++++++++++++++++++++++++++++ patchwork/forge/github/webhook.py | 35 ++++++++++++ 4 files changed, 159 insertions(+) diff --git a/patchwork/forge/github/__init__.py b/patchwork/forge/github/__init__.py index 4441c2b11ab0..ec9417b7c2e7 100644 --- a/patchwork/forge/github/__init__.py +++ b/patchwork/forge/github/__init__.py @@ -14,9 +14,13 @@ import logging from patchwork.forge import ForgeBackend from patchwork.forge import register_backend +from patchwork.forge.github.to_ml import handle_check_pending +from patchwork.forge.github.to_ml import handle_check_result from patchwork.forge.github.to_ml import handle_issue_comment from patchwork.forge.github.to_ml import handle_pull_request from patchwork.forge.github.to_ml import handle_review +from patchwork.forge.github.webhook import parse_check_run +from patchwork.forge.github.webhook import parse_check_suite from patchwork.forge.github.webhook import parse_issue_comment from patchwork.forge.github.webhook import parse_pull_request from patchwork.forge.github.webhook import parse_review @@ -44,6 +48,8 @@ class GitHubBackend(ForgeBackend): payload = json.loads(body) parsers = { + 'check_run': parse_check_run, + 'check_suite': parse_check_suite, 'issue_comment': parse_issue_comment, 'pull_request': parse_pull_request, 'pull_request_review': parse_review, @@ -79,6 +85,8 @@ class GitHubBackend(ForgeBackend): def process_webhook_event(self, forge_config, event): handlers = { + 'check_pending': handle_check_pending, + 'check_result': handle_check_result, 'issue_comment': handle_issue_comment, 'pull_request': handle_pull_request, 'review': handle_review, diff --git a/patchwork/forge/github/api.py b/patchwork/forge/github/api.py index 6e99592a9eac..36bbab596a93 100644 --- a/patchwork/forge/github/api.py +++ b/patchwork/forge/github/api.py @@ -9,6 +9,7 @@ import logging import urllib.error import urllib.request +from patchwork.forge import CheckRun from patchwork.forge import ReviewComment logger = logging.getLogger(__name__) @@ -64,3 +65,32 @@ def fetch_review_comments(gh, forge_config, pr_number, review_id): ) ) return comments + + +def fetch_check_runs(gh, forge_config, check_suite_id): + """ + Fetch check runs for a check suite via the GitHub API. + """ + owner, repo = forge_config.repo.split('/', 1) + result = gh_api_request( + gh, + forge_config, + 'GET', + f'/repos/{owner}/{repo}/check-suites/{check_suite_id}/check-runs', + ) + runs = [] + for run in result.get('check_runs', []): + status = run.get('conclusion') or run.get('status', '') + desc = '' + output = run.get('output') + if output: + desc = output.get('summary') or '' + runs.append( + CheckRun( + name=run.get('name', ''), + status=status, + url=run.get('html_url', ''), + description=desc, + ) + ) + return runs diff --git a/patchwork/forge/github/to_ml.py b/patchwork/forge/github/to_ml.py index 27797094cbe7..74c6e4d9d403 100644 --- a/patchwork/forge/github/to_ml.py +++ b/patchwork/forge/github/to_ml.py @@ -6,7 +6,12 @@ import email import logging +from django.contrib.auth import get_user_model +from django.db import transaction +from django.utils.text import slugify + from patchwork.forge.git import GitMirror +from patchwork.forge.github.api import fetch_check_runs from patchwork.forge.github.api import fetch_review_comments from patchwork.forge.util import bytes_to_mbox from patchwork.forge.util import find_series_by_pr @@ -16,6 +21,7 @@ from patchwork.forge.util import reply_to_msgid from patchwork.forge.util import sanitize_pr_body from patchwork.forge.util import send_emails from patchwork.forge.util import sender_identity +from patchwork.models import Check logger = logging.getLogger(__name__) @@ -105,6 +111,52 @@ def handle_review(gh, forge_config, event): reply(gh, forge_config, event, series, subject, body.rstrip()) +def handle_check_pending(gh, forge_config, event): + series = find_series_by_pr(gh, forge_config, event.pr_number).last() + if not series: + return + + create_checks( + gh, + forge_config, + event, + event.check_name, + Check.STATE_PENDING, + event.check_url, + '', + ) + + +def handle_check_result(gh, forge_config, event): + series = find_series_by_pr(gh, forge_config, event.pr_number).last() + if not series: + return + + check_runs = fetch_check_runs(gh, forge_config, event) + for run in check_runs: + create_checks( + series, + forge_config, + event, + run.name, + map_check_state(run.status), + run.url, + run.description, + ) + + lines = [] + for run in check_runs: + line = f'{run.name} {run.status}' + if run.url: + line += f': {run.url}' + lines.append(line) + + subject = ( + f'Re: {series.name} (GitHub: {event.check_name} {event.check_status})' + ) + reply(gh, forge_config, event, series, subject, '\n'.join(lines)) + + def reply(gh, forge_config, event, series, subject, body): """ Build a reply email as an mbox, ingest it into the database and @@ -138,3 +190,37 @@ def reply(gh, forge_config, event, series, subject, body): mbox = bytes_to_mbox(msg.as_bytes(unixfrom=True)) ingest_emails(mbox, gh, forge_config, event) send_emails(mbox, forge_config) + + +def map_check_state(conclusion): + """ + Map a GitHub check conclusion to a patchwork Check state. + """ + state_map = { + 'success': Check.STATE_SUCCESS, + 'failure': Check.STATE_FAIL, + 'timed_out': Check.STATE_FAIL, + 'cancelled': Check.STATE_FAIL, + 'action_required': Check.STATE_WARNING, + } + return state_map.get(conclusion, Check.STATE_PENDING) + + +@transaction.atomic +def create_checks(series, forge_config, event, context, state, url, desc): + # Use a dedicated 'github' user so that the checks have proper names. + user, _ = get_user_model().objects.get_or_create( + username='github', + defaults={'is_active': False}, + ) + for patch in series.patches.all(): + # New checks are created even when pending ones complete. + # The UI will deduplicate and only display the most recent ones. + Check.objects.create( + patch=patch, + user=user, + state=state, + target_url=url or '', + description=desc or '', + context=slugify(context), + ) diff --git a/patchwork/forge/github/webhook.py b/patchwork/forge/github/webhook.py index bc788512265a..17c5644eefd6 100644 --- a/patchwork/forge/github/webhook.py +++ b/patchwork/forge/github/webhook.py @@ -75,3 +75,38 @@ def parse_review(payload): body=review_body, review_state=review.get('state', ''), ) + + +def parse_check_run(payload): + if payload.get('action') != 'created': + return None + run = payload.get('check_run', {}) + prs = run.get('pull_requests', []) + if not prs: + return None + return ForgeEvent( + type='check_pending', + repo_key=get_repo_key(payload), + pr_number=prs[0].get('number', 0), + check_name=run.get('name', ''), + check_status='pending', + check_url=run.get('html_url', ''), + ) + + +def parse_check_suite(payload): + if payload.get('action') != 'completed': + return None + suite = payload.get('check_suite', {}) + prs = suite.get('pull_requests', []) + if not prs: + return None + app = suite.get('app', {}) + return ForgeEvent( + type='check_result', + repo_key=get_repo_key(payload), + pr_number=prs[0].get('number', 0), + check_suite_id=suite.get('id', 0), + check_name=app.get('name', ''), + check_status=suite.get('conclusion', ''), + ) From patchwork Thu Jun 4 14:18:22 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 7740F1BC4358 for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 13/16] forge: sync patch series to github pull requests Date: Thu, 4 Jun 2026 16:18:22 +0200 Message-ID: <20260604141826.2998337-14-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 110 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit When a patch series is completed, create a pull request on the linked forge. A signal handler on Event.CATEGORY_SERIES_COMPLETED dispatches to each project's forge backend, which applies the series mbox to a git worktree and pushes a branch. The GitHub backend creates PRs via the REST API using stdlib urllib. The PR body is built from the cover letter or first patch content, with trailers stripped and a loop prevention marker appended. Series that originated from the forge (detected by X-Patchwork-Hint: ignore in patch headers) are skipped to prevent sync loops. Series that already have PR metadata are also skipped. The signal handler is registered by load_backends() so it only fires when ENABLE_FORGE is True. Signed-off-by: Robin Jarry --- patchwork/forge/__init__.py | 45 ++++++++++++ patchwork/forge/git.py | 15 ++++ patchwork/forge/github/__init__.py | 4 ++ patchwork/forge/github/api.py | 40 +++++++++++ patchwork/forge/github/from_ml.py | 108 +++++++++++++++++++++++++++++ patchwork/forge/github/webhook.py | 14 +++- patchwork/forge/util.py | 72 +++++++++++++++++++ patchwork/settings/base.py | 3 + 8 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 patchwork/forge/github/from_ml.py diff --git a/patchwork/forge/__init__.py b/patchwork/forge/__init__.py index d9ac7486d609..cadca530daf8 100644 --- a/patchwork/forge/__init__.py +++ b/patchwork/forge/__init__.py @@ -27,6 +27,10 @@ from dataclasses import dataclass from dataclasses import field from django.conf import settings +from django.db import transaction + +from patchwork.models import Event +from patchwork.models import ForgeConfig logger = logging.getLogger(__name__) @@ -166,6 +170,16 @@ class ForgeBackend(ABC): """ raise NotImplementedError + @abstractmethod + def handle_series_completed(self, forge_config, series): + """ + Handle a completed patch series from the mailing list. + + Called by the series-completed signal handler. The backend + decides whether to create a pull request from the series. + """ + raise NotImplementedError + _backends = {} @@ -178,6 +192,37 @@ def get_backend(name): return _backends.get(name) +def _on_series_completed(sender, instance, raw, **kwargs): + if raw or instance.category != Event.CATEGORY_SERIES_COMPLETED: + return + + series = instance.series + if not series: + return + + def do_sync(): + for forge_config in ForgeConfig.objects.filter(project=series.project): + backend = get_backend(forge_config.backend) + if not backend: + continue + try: + backend.handle_series_completed(forge_config, series) + except Exception: + logger.exception( + 'forge sync failed for series %d on %s', + series.id, + forge_config.backend, + ) + + transaction.on_commit(do_sync) + + def load_backends(): + from django.db.models.signals import post_save + + from patchwork.models import Event + for module_path in settings.FORGE_BACKENDS: importlib.import_module(module_path) + + post_save.connect(_on_series_completed, sender=Event) diff --git a/patchwork/forge/git.py b/patchwork/forge/git.py index 311c04288ba9..e243960d37a7 100644 --- a/patchwork/forge/git.py +++ b/patchwork/forge/git.py @@ -272,3 +272,18 @@ class GitMirror: result = self.git(*args, capture_output=True) return bytes_to_mbox(result.stdout) + + def apply_mbox(self, mbox_text): + """ + Apply patches from an mbox string via git am -3. + """ + if isinstance(mbox_text, str): + mbox_text = mbox_text.encode('utf-8') + self.git('am', '-3', input=mbox_text) + + def push(self, branch): + """ + Force-push HEAD to refs/heads/ on the remote. + """ + with self.credentials(): + self.git('push', '-f', self.repo_url, f'HEAD:refs/heads/{branch}') diff --git a/patchwork/forge/github/__init__.py b/patchwork/forge/github/__init__.py index ec9417b7c2e7..b2dbf23a5ee7 100644 --- a/patchwork/forge/github/__init__.py +++ b/patchwork/forge/github/__init__.py @@ -14,6 +14,7 @@ import logging from patchwork.forge import ForgeBackend from patchwork.forge import register_backend +from patchwork.forge.github.from_ml import create_or_update_pr from patchwork.forge.github.to_ml import handle_check_pending from patchwork.forge.github.to_ml import handle_check_result from patchwork.forge.github.to_ml import handle_issue_comment @@ -95,5 +96,8 @@ class GitHubBackend(ForgeBackend): if handler: handler(self, forge_config, event) + def handle_series_completed(self, forge_config, series): + create_or_update_pr(self, forge_config, series) + register_backend('github', GitHubBackend()) diff --git a/patchwork/forge/github/api.py b/patchwork/forge/github/api.py index 36bbab596a93..d2332a70f0ae 100644 --- a/patchwork/forge/github/api.py +++ b/patchwork/forge/github/api.py @@ -94,3 +94,43 @@ def fetch_check_runs(gh, forge_config, check_suite_id): ) ) return runs + + +def create_pr(gh, forge_config, title, body, head, base): + """ + Create a pull request on GitHub. Return the PR number. + """ + owner, repo = forge_config.repo.split('/', 1) + result = gh_api_request( + gh, + forge_config, + 'POST', + f'/repos/{owner}/{repo}/pulls', + {'title': title, 'body': body, 'head': head, 'base': base}, + ) + return result['number'] + + +def base_branch(gh, forge_config): + """ + Return the default branch of the GitHub repository. + """ + owner, repo = forge_config.repo.split('/', 1) + result = gh_api_request( + gh, + forge_config, + 'GET', + f'/repos/{owner}/{repo}', + ) + return result['default_branch'] + + +def post_comment(gh, forge_config, pr_number, body): + owner, repo = forge_config.repo.split('/', 1) + gh_api_request( + gh, + forge_config, + 'POST', + f'/repos/{owner}/{repo}/issues/{pr_number}/comments', + {'body': body}, + ) diff --git a/patchwork/forge/github/from_ml.py b/patchwork/forge/github/from_ml.py new file mode 100644 index 000000000000..ccb44c8a78d3 --- /dev/null +++ b/patchwork/forge/github/from_ml.py @@ -0,0 +1,108 @@ +# Patchwork - automated patch tracking system +# Copyright (C) 2026 Robin Jarry +# +# SPDX-License-Identifier: GPL-2.0-or-later + + +import logging + +from patchwork.forge.git import GitMirror +from patchwork.forge.github.api import base_branch +from patchwork.forge.github.api import create_pr +from patchwork.forge.github.api import post_comment +from patchwork.forge.util import build_pr_body +from patchwork.forge.util import forge_branch_name +from patchwork.forge.util import series_from_forge +from patchwork.models import SeriesMetadata +from patchwork.views.utils import series_to_mbox + +logger = logging.getLogger(__name__) + + +def create_or_update_pr(gh, forge_config, series): + if not forge_config.sync_ml_to_forge: + return + + if SeriesMetadata.objects.filter( + series=series, key=gh.meta_key_pr() + ).exists(): + return + + if series_from_forge(series): + return + + mirror = GitMirror(gh, forge_config) + mirror.ensure_mirror() + mirror.fetch() + + base = base_branch(gh, forge_config) + pr_number, branch = find_previous_pr(gh, forge_config, series) + if not branch: + branch = forge_branch_name(series) + + mbox = series_to_mbox(series) + with mirror.worktree(base): + mirror.apply_mbox(mbox) + mirror.push(branch) + + body = build_pr_body(series) + + if pr_number: + comment = f'> Series v{series.version} submitted.\n\n{body}' + post_comment(gh, forge_config, pr_number, comment) + action = 'updated' + else: + title = series.name or 'Untitled series' + pr_number = create_pr(gh, forge_config, title, body, branch, base) + action = 'created' + + pr_ref = gh.pr_ref(forge_config, pr_number) + store_series_metadata(gh, forge_config, series, pr_ref, branch) + logger.info( + '%s PR #%d for series %d (v%d): %s', + action, + pr_number, + series.id, + series.version, + pr_ref, + ) + + +def find_previous_pr(gh, forge_config, series): + """ + Walk the previous_series chain to find a prior version that + has an existing PR. Return (pr_number, branch) or (None, None). + """ + current = series + while current.previous_series_id: + current = current.previous_series + try: + pr_meta = SeriesMetadata.objects.get( + series=current, key=gh.meta_key_pr() + ) + except SeriesMetadata.DoesNotExist: + continue + try: + branch_meta = SeriesMetadata.objects.get( + series=current, + key=f'{forge_config.backend}_branch', + ) + except SeriesMetadata.DoesNotExist: + continue + pr_url = pr_meta.value + pr_number = int(pr_url.rsplit('/', 1)[-1]) + return pr_number, branch_meta.value + return None, None + + +def store_series_metadata(gh, forge_config, series, pr_ref, branch): + SeriesMetadata.objects.update_or_create( + series=series, + key=gh.meta_key_pr(), + defaults={'value': pr_ref}, + ) + SeriesMetadata.objects.update_or_create( + series=series, + key=f'{forge_config.backend}_branch', + defaults={'value': branch}, + ) diff --git a/patchwork/forge/github/webhook.py b/patchwork/forge/github/webhook.py index 17c5644eefd6..00b8cb11d2ad 100644 --- a/patchwork/forge/github/webhook.py +++ b/patchwork/forge/github/webhook.py @@ -3,8 +3,11 @@ # # SPDX-License-Identifier: GPL-2.0-or-later +from django.conf import settings + from patchwork.forge import ForgeEvent from patchwork.forge import ForgeUser +from patchwork.forge.util import COMMENT_MARKER def parse_pull_request(payload): @@ -13,6 +16,11 @@ def parse_pull_request(payload): return None pr = payload.get('pull_request', {}) pr_body = pr.get('body') or '' + if COMMENT_MARKER in pr_body: + return None + branch = pr.get('head', {}).get('ref', '') + if branch.startswith(f'{settings.FORGE_BRANCH_PREFIX}/'): + return None return ForgeEvent( type='pull_request', repo_key=get_repo_key(payload), @@ -22,7 +30,7 @@ def parse_pull_request(payload): pr_body=pr_body, pr_head=f'pull/{pr.get("number", 0)}/head', pr_base=pr.get('base', {}).get('sha', ''), - pr_head_branch=pr.get('head', {}).get('ref', ''), + pr_head_branch=branch, pr_action=action, pr_before=payload.get('before', ''), ) @@ -51,6 +59,8 @@ def parse_issue_comment(payload): return None comment = payload.get('comment', {}) comment_body = comment.get('body') or '' + if COMMENT_MARKER in comment_body: + return None return ForgeEvent( type='issue_comment', repo_key=get_repo_key(payload), @@ -66,6 +76,8 @@ def parse_review(payload): review = payload.get('review', {}) pr = payload.get('pull_request', {}) review_body = review.get('body') or '' + if COMMENT_MARKER in review_body: + return None return ForgeEvent( type='review', repo_key=get_repo_key(payload), diff --git a/patchwork/forge/util.py b/patchwork/forge/util.py index 0793d368b7bd..114e99bbd0ae 100644 --- a/patchwork/forge/util.py +++ b/patchwork/forge/util.py @@ -14,6 +14,8 @@ import mailbox import os import re +from django.conf import settings +from django.contrib.sites.models import Site from django.core.mail import get_connection from django.db import transaction @@ -215,3 +217,73 @@ def send_emails(mbox, forge_config): ) for rcpt, err in errs.items(): logger.warning('send patch to %s failed: %s', rcpt, err) + + +TRAILER_RE = re.compile( + r'^(Signed-off-by|Acked-by|Reviewed-by|Tested-by|Reported-by' + r'|Suggested-by|Co-authored-by|Cc):.*$', + re.MULTILINE, +) + +COMMENT_MARKER = '' + + +def forge_branch_name(series): + """ + Generate a branch name for a series on the forge. + + Format: {prefix}/{slug}-{hex_id} + """ + name = series.name or '' + slug = re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') + if len(slug) > 50: + slug = slug[:50].rstrip('-') + return f'{settings.FORGE_BRANCH_PREFIX}/{slug}-{series.id:x}' + + +def build_pr_body(series): + """ + Build a pull request body from a series cover letter or first patch. + + Strips git trailers and appends a patchwork link with submitter + attribution and a loop prevention marker. + """ + if series.cover_letter and series.cover_letter.content: + body = series.cover_letter.content.strip() + else: + patches = list(series.patches.order_by('number')[:1]) + if patches and patches[0].content: + body = patches[0].content.strip() + else: + body = '' + + body = TRAILER_RE.sub('', body).strip() + + submitter = series.submitter + name = submitter.name or submitter.email + site = Site.objects.get_current() + project = series.project.linkname + url = f'https://{site.domain}/project/{project}/list/?series={series.id}' + + body += f'\n\n> Submitted by {name} on the mailing list.' + body += f'\n> [View on Patchwork]({url})' + body += f'\n\n{COMMENT_MARKER}' + return body + + +def series_from_forge(series): + """ + Return True if the series originated from a forge sync. + + Checks for X-Patchwork-Hint: ignore in patch headers, which is + added by format_patches() when sending forge PRs to the mailing + list. + """ + for patch in series.patches.all()[:1]: + if not patch.headers: + continue + parsed = email.message_from_string(patch.headers) + hint = parsed.get('X-Patchwork-Hint', '') + if hint.lower() == 'ignore': + return True + return False diff --git a/patchwork/settings/base.py b/patchwork/settings/base.py index bc42b14b2242..539ab83e3c3c 100644 --- a/patchwork/settings/base.py +++ b/patchwork/settings/base.py @@ -306,3 +306,6 @@ FORGE_AUTH = {} # Base directory for git mirror clones (one bare repo per project) FORGE_GIT_MIRROR_PATH = '' + +# Branch prefix for forge-created branches (loop prevention) +FORGE_BRANCH_PREFIX = 'patchwork' From patchwork Thu Jun 4 14:18:23 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id 8F4FC1BC4359 for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 14/16] forge: forward mailing list comments to forge pull requests Date: Thu, 4 Jun 2026 16:18:23 +0200 Message-ID: <20260604141826.2998337-15-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 111 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add a signal handler for patch and cover letter comment events that forwards them to the linked forge pull request. Comments are posted with the original author name (without email address) and the comment body formatted as a GitHub quote block. Comments that originated from the forge (detected by X-Patchwork-Hint: ignore in the comment headers) are skipped to prevent sync loops. Signed-off-by: Robin Jarry --- patchwork/forge/__init__.py | 53 +++++++++++++++++++++++++++--- patchwork/forge/github/__init__.py | 4 +++ patchwork/forge/github/api.py | 3 +- patchwork/forge/github/from_ml.py | 25 ++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/patchwork/forge/__init__.py b/patchwork/forge/__init__.py index cadca530daf8..79a136602d6e 100644 --- a/patchwork/forge/__init__.py +++ b/patchwork/forge/__init__.py @@ -28,6 +28,7 @@ from dataclasses import field from django.conf import settings from django.db import transaction +from django.db.models.signals import post_save from patchwork.models import Event from patchwork.models import ForgeConfig @@ -180,6 +181,16 @@ class ForgeBackend(ABC): """ raise NotImplementedError + @abstractmethod + def handle_comment_created(self, forge_config, comment, series): + """ + Handle a comment on a patch or cover letter. + + Called by the comment-created signal handler. The backend + decides whether to forward the comment to the forge PR. + """ + raise NotImplementedError + _backends = {} @@ -217,12 +228,46 @@ def _on_series_completed(sender, instance, raw, **kwargs): transaction.on_commit(do_sync) +def _on_comment_created(sender, instance, raw, **kwargs): + if raw: + return + + if instance.category == Event.CATEGORY_PATCH_COMMENT_CREATED: + comment = instance.patch_comment + if not comment: + return + series = comment.patch.series + elif instance.category == Event.CATEGORY_COVER_COMMENT_CREATED: + comment = instance.cover_comment + if not comment: + return + series = comment.cover.series + else: + return + + if not series: + return + + def do_sync(): + for forge_config in ForgeConfig.objects.filter(project=series.project): + backend = get_backend(forge_config.backend) + if not backend: + continue + try: + backend.handle_comment_created(forge_config, comment, series) + except Exception: + logger.exception( + 'forge comment sync failed for series %d on %s', + series.id, + forge_config.backend, + ) + + transaction.on_commit(do_sync) + + def load_backends(): - from django.db.models.signals import post_save - - from patchwork.models import Event - for module_path in settings.FORGE_BACKENDS: importlib.import_module(module_path) post_save.connect(_on_series_completed, sender=Event) + post_save.connect(_on_comment_created, sender=Event) diff --git a/patchwork/forge/github/__init__.py b/patchwork/forge/github/__init__.py index b2dbf23a5ee7..18200a4fdb23 100644 --- a/patchwork/forge/github/__init__.py +++ b/patchwork/forge/github/__init__.py @@ -15,6 +15,7 @@ import logging from patchwork.forge import ForgeBackend from patchwork.forge import register_backend from patchwork.forge.github.from_ml import create_or_update_pr +from patchwork.forge.github.from_ml import post_pr_comment from patchwork.forge.github.to_ml import handle_check_pending from patchwork.forge.github.to_ml import handle_check_result from patchwork.forge.github.to_ml import handle_issue_comment @@ -99,5 +100,8 @@ class GitHubBackend(ForgeBackend): def handle_series_completed(self, forge_config, series): create_or_update_pr(self, forge_config, series) + def handle_comment_created(self, forge_config, comment, series): + post_pr_comment(self, forge_config, comment, series) + register_backend('github', GitHubBackend()) diff --git a/patchwork/forge/github/api.py b/patchwork/forge/github/api.py index d2332a70f0ae..92b4a73bc161 100644 --- a/patchwork/forge/github/api.py +++ b/patchwork/forge/github/api.py @@ -11,6 +11,7 @@ import urllib.request from patchwork.forge import CheckRun from patchwork.forge import ReviewComment +from patchwork.forge.util import COMMENT_MARKER logger = logging.getLogger(__name__) @@ -55,7 +56,7 @@ def fetch_review_comments(gh, forge_config, pr_number, review_id): comments = [] for r in results: body = r.get('body') or '' - if body == '': + if body == '' or COMMENT_MARKER in body: continue comments.append( ReviewComment( diff --git a/patchwork/forge/github/from_ml.py b/patchwork/forge/github/from_ml.py index ccb44c8a78d3..4b1acf4f424c 100644 --- a/patchwork/forge/github/from_ml.py +++ b/patchwork/forge/github/from_ml.py @@ -4,12 +4,14 @@ # SPDX-License-Identifier: GPL-2.0-or-later +import email import logging from patchwork.forge.git import GitMirror from patchwork.forge.github.api import base_branch from patchwork.forge.github.api import create_pr from patchwork.forge.github.api import post_comment +from patchwork.forge.util import COMMENT_MARKER from patchwork.forge.util import build_pr_body from patchwork.forge.util import forge_branch_name from patchwork.forge.util import series_from_forge @@ -106,3 +108,26 @@ def store_series_metadata(gh, forge_config, series, pr_ref, branch): key=f'{forge_config.backend}_branch', defaults={'value': branch}, ) + + +def post_pr_comment(gh, forge_config, comment, series): + if not forge_config.sync_ml_to_forge: + return + + if comment.headers: + parsed = email.message_from_string(comment.headers) + hint = parsed.get('X-Patchwork-Hint', '') + if hint.lower() == 'ignore': + return + + pr_meta = SeriesMetadata.objects.filter( + series=series, key=gh.meta_key_pr() + ).first() + if not pr_meta: + return + + pr_number = int(pr_meta.value.rsplit('/', 1)[-1]) + author = comment.submitter.name or comment.submitter.email + quoted = '\n'.join(f'> {line}' for line in comment.content.splitlines()) + body = f'**{author}** commented:\n\n{quoted}\n\n{COMMENT_MARKER}' + post_comment(gh, forge_config, pr_number, body) From patchwork Thu Jun 4 14:18:24 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id C0E191BC435A for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 15/16] docs: add forge integration documentation Date: Thu, 4 Jun 2026 16:18:24 +0200 Message-ID: <20260604141826.2998337-16-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 112 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Add user-facing documentation for the forge sync feature covering configuration, per-project setup, webhook endpoints, the GitHub backend, loop prevention mechanisms, series metadata, and email headers. Reference the new forge settings from the deployment configuration guide. Signed-off-by: Robin Jarry --- docs/deployment/configuration.rst | 37 ++++ docs/usage/forge.rst | 294 ++++++++++++++++++++++++++++++ docs/usage/index.rst | 1 + 3 files changed, 332 insertions(+) create mode 100644 docs/usage/forge.rst diff --git a/docs/deployment/configuration.rst b/docs/deployment/configuration.rst index a71dd3f4bb57..958144b12ecb 100644 --- a/docs/deployment/configuration.rst +++ b/docs/deployment/configuration.rst @@ -73,6 +73,13 @@ This is customizable on a per-user basis from the user configuration page. This option was previously named ``DEFAULT_PATCHES_PER_PAGE``. It was renamed as cover letters are now supported also. +``ENABLE_FORGE`` +~~~~~~~~~~~~~~~~ + +Enable the :doc:`forge integration <../usage/forge>`. When enabled, webhook +endpoints are registered and sync signal handlers are active. Requires +``FORGE_BACKENDS`` to list at least one backend module. + ``ENABLE_REST_API`` ~~~~~~~~~~~~~~~~~~~ @@ -87,6 +94,36 @@ Enable the :doc:`XML-RPC API <../api/xmlrpc>`. .. TODO(stephenfin) Deprecate this in favor of SECURE_SSL_REDIRECT +``FORGE_AUTH`` +~~~~~~~~~~~~~~ + +Authentication credentials for forge backends. See +:doc:`forge integration <../usage/forge>` for details. + +``FORGE_BACKENDS`` +~~~~~~~~~~~~~~~~~~ + +List of forge backend modules to load. See +:doc:`forge integration <../usage/forge>` for details. + +``FORGE_BRANCH_PREFIX`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Branch prefix for forge-created branches. Used for loop prevention. Default: +``'patchwork'``. See :doc:`forge integration <../usage/forge>` for details. + +``FORGE_GIT_MIRROR_PATH`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Base directory for bare git mirror clones. See +:doc:`forge integration <../usage/forge>` for details. + +``FORGE_WEBHOOK_SECRETS`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Webhook signature verification secrets per backend. See +:doc:`forge integration <../usage/forge>` for details. + ``FORCE_HTTPS_LINKS`` ~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/usage/forge.rst b/docs/usage/forge.rst new file mode 100644 index 000000000000..9ee1e9794f13 --- /dev/null +++ b/docs/usage/forge.rst @@ -0,0 +1,294 @@ +Forge Integration +================= + +Patchwork can synchronize patch workflows with code forges such as GitHub. +When enabled, patch series submitted to a mailing list are automatically +converted into pull requests, and pull requests opened on a forge are converted +into patch series on the mailing list. Comments, reviews, and CI results flow +in both directions. + +.. contents:: + :local: + +Overview +-------- + +The forge integration provides bidirectional synchronization: + +**Mailing list to forge:** + +- A completed patch series creates a pull request on the forge. +- Respin submissions (v2, v3, ...) force-push to the same branch and post an + update comment on the existing pull request. +- Comments on patches or cover letters are forwarded to the pull request. + +**Forge to mailing list:** + +- A pull request opened on the forge generates a patch series email sent to the + mailing list and ingested directly into the Patchwork database. +- Respin (force-push) events generate a new version of the patch series with + proper version numbering and optional threading to the original. +- Comments and reviews on the pull request are forwarded as email replies to the + series. +- CI check results are recorded as Patchwork checks on each patch and + summarized in an email to the list. + +Loop prevention ensures that events originating from the sync itself are not +re-synced in the other direction. + +Enabling Forge Support +---------------------- + +Forge support is gated behind the ``ENABLE_FORGE`` setting. To enable it, add +the following to your ``settings.py``: + +.. code-block:: python + + ENABLE_FORGE = True + FORGE_BACKENDS = ['patchwork.forge.github'] + +Each backend module is only imported when listed in ``FORGE_BACKENDS``. +Backend-specific dependencies (if any) are only required when the backend is +enabled. + +Settings Reference +------------------ + +``ENABLE_FORGE`` +~~~~~~~~~~~~~~~~ + +Set to ``True`` to enable forge integration. When disabled, no webhook +endpoints are registered and no sync signal handlers are active. + +Default: ``False`` + +``FORGE_BACKENDS`` +~~~~~~~~~~~~~~~~~~ + +A list of Python module paths for forge backends to load at startup. Each +module must call ``register_backend()`` to register itself. + +Default: ``[]`` + +Example: + +.. code-block:: python + + FORGE_BACKENDS = ['patchwork.forge.github'] + +``FORGE_WEBHOOK_SECRETS`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +A dictionary mapping backend names to their webhook secrets. Used for +HMAC-SHA256 signature verification of incoming webhooks. If a backend's secret +is empty, signature verification is skipped. + +Default: ``{}`` + +Example: + +.. code-block:: python + + FORGE_WEBHOOK_SECRETS = { + 'github': 'your-webhook-secret', + } + +``FORGE_AUTH`` +~~~~~~~~~~~~~~ + +Authentication credentials for forge backends. Each backend gets a top-level +dictionary with optional per-repository overrides under a ``repos`` key. + +Default: ``{}`` + +Example: + +.. code-block:: python + + FORGE_AUTH = { + 'github': { + 'token': 'ghp_default_token', + 'repos': { + 'owner/special-repo': { + 'token': 'ghp_different_token', + }, + }, + }, + } + +When resolving credentials for a project, the backend-level defaults are merged +with any repository-specific overrides. This allows most projects to share the +same bot account while specific repositories use different credentials. + +``FORGE_GIT_MIRROR_PATH`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Base directory for bare git mirror clones. One mirror is created per project +at ``{FORGE_GIT_MIRROR_PATH}/{project_linkname}.git``. + +Default: ``''`` + +Example: + +.. code-block:: python + + FORGE_GIT_MIRROR_PATH = '/var/cache/patchwork/mirrors' + +``FORGE_BRANCH_PREFIX`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Prefix for branches created by patchwork on the forge. Used for loop +prevention: pull requests on branches starting with this prefix are recognized +as patchwork-created and skipped during forge-to-mailing-list sync. + +Default: ``'patchwork'`` + +Per-Project Configuration +------------------------- + +Each Patchwork project can be linked to one or more forge repositories through +the ``ForgeConfig`` model, configurable via the Django admin interface as an +inline on the Project admin page. + +**Fields:** + +``backend`` + The forge backend name (e.g. ``github``). Must match a registered backend. + +``repo`` + The repository identifier on the forge (e.g. ``owner/repo`` for GitHub). + +``from_email`` + Sender address for forge-originated emails. Falls back to + ``DEFAULT_FROM_EMAIL`` if empty. + +``sync_ml_to_forge`` + Enable creating pull requests from mailing list patch series. Default: + ``True``. + +``sync_forge_to_ml`` + Enable sending patch emails from forge pull requests to the mailing list. + Default: ``True``. + +``thread_respins`` + When enabled, respin series (v2, v3, ...) from the forge are threaded as + replies to the original version's cover letter. When disabled, each version + starts its own thread. Default: ``False``. + +Webhook Setup +------------- + +Each forge backend exposes a webhook endpoint at:: + + /webhook/forge// + +For example, with the GitHub backend enabled:: + + https://your-patchwork-instance.example.com/webhook/forge/github/ + +Configure this URL as the webhook endpoint on your forge, selecting the +events you want to synchronize. + +GitHub Backend +-------------- + +The GitHub backend (``patchwork.forge.github``) supports the following webhook +events: + +- ``pull_request`` (opened, synchronize) +- ``issue_comment`` (created) +- ``pull_request_review`` (submitted) +- ``check_run`` (created) +- ``check_suite`` (completed) + +**Required GitHub webhook configuration:** + +- Content type: ``application/json`` +- Secret: must match ``FORGE_WEBHOOK_SECRETS['github']`` +- Events: select the events listed above, or choose "Send me everything" + +**Required permissions** (for a GitHub App or Personal Access Token): + +- ``contents:read`` — to fetch pull request refs +- ``pull_requests:write`` — to create pull requests and post comments +- ``checks:read`` — to receive check run/suite events + +**Authentication:** + +The GitHub backend uses a Personal Access Token configured in ``FORGE_AUTH``: + +.. code-block:: python + + FORGE_AUTH = { + 'github': { + 'token': 'ghp_your_token_here', + }, + } + +The token is used for both API calls (creating PRs, posting comments) and git +operations (cloning, fetching, pushing). + +Loop Prevention +--------------- + +Three mechanisms prevent infinite sync loops: + +**Branch prefix check:** + Pull requests on branches starting with ``FORGE_BRANCH_PREFIX`` (default: + ``patchwork/``) are recognized as patchwork-created and skipped during + forge-to-mailing-list sync. This check is done at webhook parsing time. + +**HTML comment marker:** + All pull request bodies and comments posted by patchwork contain the + ```` HTML comment. Webhook events containing this marker + are skipped during parsing. + +**Email header check:** + All emails sent by the forge sync include an ``X-Patchwork-Hint: ignore`` + header. When these emails return through the mail pipeline, ``parsemail`` + skips them. When processing mailing-list-to-forge sync, comments with this + header are not forwarded to the forge. + +Series Metadata +--------------- + +When a patch series is linked to a forge pull request, two metadata entries are +stored on the series: + +- ``{backend}_pr`` — the full URL to the pull request (e.g. + ``https://github.com/owner/repo/pull/42``) +- ``{backend}_branch`` — the branch name on the forge (e.g. + ``patchwork/2a/fix-memory-leak``) + +This metadata is used to find existing pull requests for respin detection and +to route comments and check results to the correct pull request. + +Email Headers +------------- + +Emails generated by the forge-to-mailing-list sync include the following +custom headers: + +``X-Patchwork-Hint: ignore`` + Tells ``parsemail`` to skip this email when it returns through the mail + pipeline (since it was already ingested directly). + +``Sender`` + The patchwork bot address (from ``ForgeConfig.from_email`` or + ``DEFAULT_FROM_EMAIL``). + +``List-ID`` + The project's list ID, ensuring correct project routing. + +``Reply-To`` + The project's mailing list address. + +For patch series from pull requests, additional headers are added via +``git format-patch``: + +``To`` + The project's mailing list address. + +``Cc`` + Addresses extracted from commit trailers (Signed-off-by, Acked-by, + Reviewed-by, Tested-by, etc.). diff --git a/docs/usage/index.rst b/docs/usage/index.rst index 9d7dcf019854..16aea998bd41 100644 --- a/docs/usage/index.rst +++ b/docs/usage/index.rst @@ -16,4 +16,5 @@ with overviews of specific features. /usage/design /usage/delegation /usage/headers + /usage/forge /usage/clients From patchwork Thu Jun 4 14:18:25 2026 Return-Path: Received: from ringo (2a01cb00021ec0002e23edbec21b0e73.ipv6.abo.wanadoo.fr [IPv6:2a01:cb00:21e:c000:2e23:edbe:c21b:e73]) by patches.jarry.cc (Postfix) with ESMTP id D713C1BC435B for ; Thu, 04 Jun 2026 16:18:30 +0200 (CEST) From: Robin Jarry To: pw@patches.jarry.cc Subject: [PATCH v3 16/16] deploy Date: Thu, 4 Jun 2026 16:18:25 +0200 Message-ID: <20260604141826.2998337-17-robin@jarry.cc> X-Mailer: git-send-email 2.54.0 In-Reply-To: <20260604141826.2998337-1-robin@jarry.cc> References: <20260604141826.2998337-1-robin@jarry.cc> MIME-Version: 1.0 List-ID: X-Patchwork-Submitter: Robin Jarry X-Patchwork-Id: 113 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Signed-off-by: Robin Jarry --- Makefile | 147 +++++++++++++++++++++++++++++++ nginx.conf | 62 +++++++++++++ parsemail.sh | 14 +++ patchwork.service | 22 +++++ patchwork/settings/production.py | 67 ++++++++++++++ postfix/client_access | 2 + postfix/header_checks | 2 + postfix/main.cf | 53 +++++++++++ postfix/master.cf | 28 ++++++ postfix/recipient_bcc | 1 + postfix/transport | 2 + uwsgi.ini | 10 +++ 12 files changed, 410 insertions(+) create mode 100644 Makefile create mode 100644 nginx.conf create mode 100755 parsemail.sh create mode 100644 patchwork.service create mode 100644 patchwork/settings/production.py create mode 100644 postfix/client_access create mode 100644 postfix/header_checks create mode 100644 postfix/main.cf create mode 100644 postfix/master.cf create mode 100644 postfix/recipient_bcc create mode 100644 postfix/transport create mode 100644 uwsgi.ini diff --git a/Makefile b/Makefile new file mode 100644 index 000000000000..32a8167ae9bb --- /dev/null +++ b/Makefile @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 Robin Jarry +# +# Makefile for deploying pwforge + patchwork on a single host. +# Run as root or with sudo. + +DOMAIN ?= patches.jarry.cc +DB_NAME ?= patchwork +DB_USER ?= patchwork +DB_PASSWORD ?= $(shell openssl rand -base64 24) +DJANGO_SECRET ?= $(shell openssl rand -base64 48) +WEBHOOK_SECRET ?= $(shell openssl rand -base64 48) + +.PHONY: help +help: + @echo "usage: make [VAR=value ...]" + @echo "" + @echo "targets:" + @echo " deps install system packages" + @echo " database create postgresql database" + @echo " patchwork install patchwork" + @echo " uwsgi install uwsgi config and service" + @echo " postfix install postfix config" + @echo " nginx install nginx config and obtain TLS cert" + @echo " install install everything (idempotent)" + @echo "" + @echo "variables:" + @echo " DOMAIN $(DOMAIN)" + @echo " DB_NAME $(DB_NAME)" + @echo " DB_USER $(DB_USER)" + @echo " DB_PASSWORD (auto-generated if unset)" + @echo " DJANGO_SECRET (auto-generated if unset)" + +DEB := git nginx certbot python3-certbot-nginx postfix opendkim opendkim-tools +DEB += uwsgi uwsgi-plugin-python3 +DEB += postgresql python3-psycopg2 +DEB += python3-django python3-djangorestframework python3-django-filters + +.PHONY: deps +deps: + apt-get update -q + apt-get install -qy --no-install-recommends $(DEB) + usermod -a -G opendkim postfix + +/etc/default/patchwork: + @echo "DJANGO_SETTINGS_MODULE=patchwork.settings.production" > $@ + @echo "DJANGO_SECRET_KEY=$(DJANGO_SECRET)" >> $@ + @echo "DATABASE_NAME=$(DB_NAME)" >> $@ + @echo "DATABASE_USER=$(DB_USER)" >> $@ + @echo "DATABASE_PASSWORD=$(DB_PASSWORD)" >> $@ + @echo "DATABASE_HOST=localhost" >> $@ + @echo "DATABASE_PORT=" >> $@ + @echo "STATIC_ROOT=/var/www/patchwork" >> $@ + @echo "ALLOWED_HOSTS=$(DOMAIN)" >> $@ + @echo "FROM_EMAIL='Patchwork '" >> $@ + @echo "GH_WEBHOOK_SECRET=$(WEBHOOK_SECRET)" >> $@ + @echo "GH_TOKEN=replace_me" >> $@ + chmod 640 $@ + chgrp www-data $@ + +.PHONY: database +database: /etc/default/patchwork + @set -e; . $<; \ + user=`sudo -u postgres psql -tXAc "SELECT 1 FROM pg_roles WHERE rolname='$$DATABASE_USER';"`; \ + if [ -z "$$user" ]; then \ + sudo -u postgres psql -c "CREATE USER $$DATABASE_USER PASSWORD '$$DATABASE_PASSWORD';"; \ + sudo -u postgres createdb -O $$DATABASE_USER $$DATABASE_NAME; \ + fi + +.PHONY: patchwork +patchwork: database + mkdir -p /var/cache/patchwork/mirrors + chmod 770 /var/cache/patchwork/mirrors + chgrp -R www-data /var/cache/patchwork/mirrors + mkdir -p /var/www/patchwork + set -a && . /etc/default/patchwork && set +a -xe && \ + python3 manage.py migrate --noinput && \ + python3 manage.py collectstatic --noinput && \ + python3 manage.py loaddata default_tags default_states + +/etc/uwsgi/apps-available/patchwork.ini: uwsgi.ini + install -Dm 0644 $< $@ + sed -i 's,^chdir.*,chdir = $(CURDIR),' $@ + +/etc/uwsgi/apps-enabled/patchwork.ini: /etc/uwsgi/apps-available/patchwork.ini + ln -sfr $< $@ + +/etc/systemd/system/%.service: %.service + install -Dm 0644 $< $@ + +.PHONY: uwsgi +uwsgi: /etc/uwsgi/apps-enabled/patchwork.ini /etc/systemd/system/patchwork.service + +/etc/postfix/%: postfix/% + install -D -m 0644 $< $@ + sed -i 's/patches.example.com/$(DOMAIN)/g' $@ + sed -i 's,/opt/patchwork,$(CURDIR),' $@ + +/etc/dkimkeys/$(DOMAIN)/mail.private: + mkdir -p $(@D) + opendkim-genkey -d $(DOMAIN) -s mail -D $(@D) + chown -R opendkim:opendkim $(@D) + sed -Ei '/^(Domain|Selector|KeyFile)/d' /etc/opendkim.conf + echo 'Domain $(DOMAIN)' >> /etc/opendkim.conf + echo 'Selector mail' >> /etc/opendkim.conf + echo 'KeyFile $@' >> /etc/opendkim.conf + +.PHONY: postfix +postfix: $(addprefix /etc/,$(wildcard postfix/*)) /etc/dkimkeys/$(DOMAIN)/mail.private + postmap /etc/postfix/transport + postmap /etc/postfix/recipient_bcc + +/etc/nginx/sites-enabled/$(DOMAIN): nginx.conf + @if ! [ -r "/etc/letsencrypt/live/$(DOMAIN)/cert.pem" ]; then \ + set -xe; \ + rm -f $@ /etc/nginx/sites-enabled/pw_; \ + awk '/^server / {count++} count<2' $< | \ + sed 's/patches.example.com/$(DOMAIN)/g' > \ + /etc/nginx/sites-enabled/pw_; \ + systemctl reload nginx; \ + mkdir -p /var/www/acme; \ + certbot certonly --webroot -w /var/www/acme -d $(DOMAIN); \ + rm -f /etc/nginx/sites-enabled/pw_; \ + fi + install -Dm 644 $< /etc/nginx/sites-available/$(DOMAIN) + sed -i 's/patches.example.com/$(DOMAIN)/g' /etc/nginx/sites-available/$(DOMAIN) + ln -sfr /etc/nginx/sites-available/$(DOMAIN) $@ + nginx -t + +.PHONY: nginx +nginx: /etc/nginx/sites-enabled/$(DOMAIN) + +.PHONY: install +install: database patchwork uwsgi postfix nginx + systemctl daemon-reload + systemctl enable --now patchwork postfix nginx opendkim + systemctl restart patchwork postfix nginx opendkim + @echo + @echo "Update your DNS zone with the following entries:" + @echo + @ip -j addr | jq -r '.[].addr_info[] | select(.scope == "global" and .family == "inet") | .local' | \ + sed 's/^/$(DOMAIN). 10800 IN A /' + @ip -j addr | jq -r '.[].addr_info[] | select(.scope == "global" and .family == "inet6") | .local' | \ + sed 's/^/$(DOMAIN). 10800 IN AAAA /' + @echo "$(DOMAIN). 10800 IN MX 10 $(DOMAIN)." + @echo '$(DOMAIN). 10800 IN TXT "v=spf1 a -all"' + @sed -E 's/^(mail._domainkey)/\1.$(DOMAIN)./' /etc/dkimkeys/$(DOMAIN)/mail.txt diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 000000000000..1be8d5c1aabe --- /dev/null +++ b/nginx.conf @@ -0,0 +1,62 @@ +# /etc/nginx/sites-available/pwforge +# +# Reverse proxy for patchwork and pwforge behind nginx. +# Use certbot to obtain TLS certificates: +# certbot certonly --webroot -w /var/www/acme -d patches.example.com + +limit_req_zone $pw_limit zone=pw:10m rate=20r/s; + +# Only rate-limit write requests (POST, PUT, DELETE) +map $request_method $pw_limit { + GET ""; + HEAD ""; + default $binary_remote_addr; +} + +server { + listen 80; + listen [::]:80; + server_name patches.example.com; + + location /.well-known/acme-challenge/ { + root /var/www/acme; + } + location / { + return 301 https://patches.example.com$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name patches.example.com; + + ssl_certificate /etc/letsencrypt/live/patches.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/patches.example.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + limit_req zone=pw burst=10 nodelay; + + server_tokens off; + + location = favicon.ico { + access_log off; + log_not_found off; + } + + location /static { + alias /var/www/patchwork; + expires 3h; + gzip on; + gzip_types text/css application/javascript; + gzip_min_length 500; + gzip_disable msie6; + } + + # patchwork web UI and API + location / { + include uwsgi_params; + uwsgi_pass unix:/run/patchwork/uwsgi.sock; + } +} diff --git a/parsemail.sh b/parsemail.sh new file mode 100755 index 000000000000..a597b4ec1e05 --- /dev/null +++ b/parsemail.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2026 Robin Jarry + +# Called by postfix pipe transport to feed incoming mail to patchwork. +# Exit 75 (EX_TEMPFAIL) on failure so postfix retries later. + +set -a +. /etc/default/patchwork +set +a + +cd $(dirname $0) +systemd-cat --identifier=parsemail --priority=debug --stderr-priority=err \ + python3 manage.py parsemail || exit 75 diff --git a/patchwork.service b/patchwork.service new file mode 100644 index 000000000000..3b8c3102473b --- /dev/null +++ b/patchwork.service @@ -0,0 +1,22 @@ +[Unit] +Description=Patchwork uWSGI +After=postgresql.service + +[Service] +ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/apps-enabled/patchwork.ini +Restart=always +KillSignal=SIGQUIT +Type=notify +NotifyAccess=all +User=www-data +Group=www-data +EnvironmentFile=/etc/default/patchwork +ReadWritePaths=/var/cache/patchwork/mirrors +RuntimeDirectory=patchwork +ProtectSystem=strict +ProtectHome=true +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/patchwork/settings/production.py b/patchwork/settings/production.py new file mode 100644 index 000000000000..647a2585e0f4 --- /dev/null +++ b/patchwork/settings/production.py @@ -0,0 +1,67 @@ +""" +Patchwork production settings for forge sync deployment. +""" + +import os + +from .base import * # noqa + +DEFAULT_FROM_EMAIL = os.environ.get("FROM_EMAIL", DEFAULT_FROM_EMAIL) +SERVER_EMAIL = DEFAULT_FROM_EMAIL +NOTIFICATION_FROM_EMAIL = DEFAULT_FROM_EMAIL +EMAIL_SUBJECT_PREFIX = "[patchwork] " + +SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] + +ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "localhost").split(",") + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "HOST": os.environ.get("DATABASE_HOST", "localhost"), + "PORT": os.environ.get("DATABASE_PORT", ""), + "NAME": os.environ.get("DATABASE_NAME", "patchwork"), + "USER": os.environ.get("DATABASE_USER", "patchwork"), + "PASSWORD": os.environ.get("DATABASE_PASSWORD", ""), + }, +} + +STATIC_ROOT = os.environ.get("STATIC_ROOT", "/var/www/patchwork") + +STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" + +TIME_ZONE = "UTC" + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "syslog": { + "format": "%(name)s: %(message)s", + }, + }, + "handlers": { + "syslog": { + "class": "logging.handlers.SysLogHandler", + "address": "/dev/log", + "facility": "daemon", + "formatter": "syslog", + }, + }, + "root": { + "handlers": ["syslog"], + "level": "INFO", + }, +} + +ENABLE_FORGE = True +FORGE_BACKENDS = ["patchwork.forge.github"] +FORGE_WEBHOOK_SECRETS = { + "github": os.environ.get("GH_WEBHOOK_SECRET", ""), +} +FORGE_GIT_MIRROR_PATH = "/var/cache/patchwork/mirrors" +FORGE_AUTH = { + "github": { + "token": os.environ.get("GH_TOKEN", ""), + }, +} diff --git a/postfix/client_access b/postfix/client_access new file mode 100644 index 000000000000..91907452942b --- /dev/null +++ b/postfix/client_access @@ -0,0 +1,2 @@ +# mails.dpdk.org +217.70.189.124/32 OK diff --git a/postfix/header_checks b/postfix/header_checks new file mode 100644 index 000000000000..751678685afb --- /dev/null +++ b/postfix/header_checks @@ -0,0 +1,2 @@ +/^From:.*pw@patches\.jarry\.cc/ PREPEND List-Id: +/^To:.*pw@patches\.jarry\.cc/ PREPEND List-Id: diff --git a/postfix/main.cf b/postfix/main.cf new file mode 100644 index 000000000000..d5160981bdaa --- /dev/null +++ b/postfix/main.cf @@ -0,0 +1,53 @@ +# Postfix configuration for pwforge patchwork mail ingestion. +# +# This instance only accepts mail from a designated upstream MTA and +# delivers it to patchwork via a pipe transport. It does not relay +# or deliver mail to local users. + +compatibility_level = 3.6 + +# Replace with your actual hostname and domain. +myhostname = patches.jarry.cc +mydomain = patches.jarry.cc +myorigin = $mydomain +mydestination = $myhostname, localhost + +inet_interfaces = all +inet_protocols = all + +# Only accept mail from the upstream MTA and localhost (for pwforge's +# git send-email). Replace CHANGE_ME with the upstream MTA address. +mynetworks = 127.0.0.0/8, [::1]/128 +smtpd_client_restrictions = + permit_mynetworks, + check_client_access cidr:/etc/postfix/client_access, + reject + +# Route list mail to patchwork parsemail pipe. +header_checks = regexp:/etc/postfix/header_checks +recipient_bcc_maps = hash:/etc/postfix/recipient_bcc +transport_maps = hash:/etc/postfix/transport + +# Discard mail to unknown local users silently. +local_recipient_maps = +fallback_transport = discard: + +# Do not relay email, only allow local send +smtpd_relay_restrictions = permit_mynetworks, reject_unauth_destination +default_transport = smtp: +relay_domains = +relayhost = + +# Generous limits for patch series. +message_size_limit = 52428800 +mailbox_size_limit = 0 + +# Retry quickly on transient failures (e.g. database locks). +minimal_backoff_time = 30s +maximal_backoff_time = 300s +queue_run_delay = 30s + +# DKIM +milter_default_action = accept +smtpd_milters = unix:/run/opendkim/opendkim.sock +non_smtpd_milters = unix:/run/opendkim/opendkim.sock diff --git a/postfix/master.cf b/postfix/master.cf new file mode 100644 index 000000000000..54e8e0b248ab --- /dev/null +++ b/postfix/master.cf @@ -0,0 +1,28 @@ +# Postfix master process configuration for pwforge patchwork ingestion. + +smtp inet n - n - - smtpd +pickup unix n - n 60 1 pickup +cleanup unix n - n - 0 cleanup +qmgr unix n - n 300 1 qmgr +tlsmgr unix - - n 1000? 1 tlsmgr +rewrite unix - - n - - trivial-rewrite +bounce unix - - n - 0 bounce +defer unix - - n - 0 bounce +trace unix - - n - 0 bounce +verify unix - - n - 1 verify +flush unix n - n 1000? 0 flush +proxymap unix - - n - - proxymap +proxywrite unix - - n - 1 proxymap +smtp unix - - n - - smtp +relay unix - - n - - smtp -o syslog_name=postfix/$service_name +showq unix n - n - - showq +error unix - - n - - error +retry unix - - n - - error +discard unix - - n - - discard +local unix - n n - - local +virtual unix - n n - - virtual +lmtp unix - - n - - lmtp +anvil unix - - n - 1 anvil +scache unix - - n - 1 scache +postlog unix-dgram n - n - 1 postlogd +patchwork unix - n n - 1 pipe flags=FR user=www-data argv=/opt/patchwork/parsemail.sh diff --git a/postfix/recipient_bcc b/postfix/recipient_bcc new file mode 100644 index 000000000000..7c6c2230904c --- /dev/null +++ b/postfix/recipient_bcc @@ -0,0 +1 @@ +pw@patches.jarry.cc robin@jarry.cc diff --git a/postfix/transport b/postfix/transport new file mode 100644 index 000000000000..742b403310c3 --- /dev/null +++ b/postfix/transport @@ -0,0 +1,2 @@ +# Route mail for the subscribed address to patchwork +patches.jarry.cc patchwork: diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 000000000000..12811e2c63d9 --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,10 @@ +[uwsgi] +plugins = python3 +chdir = /opt/patchwork +module = patchwork.wsgi:application +master = true +processes = 4 +socket = /run/patchwork/uwsgi.sock +chmod-socket = 660 +buffer-size = 16384 +vacuum = true