[Python-checkins] release: Major updates for my local 3.4 build tool.
larry.hastings
python-checkins at python.org
Wed Feb 19 04:49:25 CET 2014
http://hg.python.org/release/rev/77e1e7cfbc7d
changeset: 72:77e1e7cfbc7d
user: Larry Hastings <larry at hastings.org>
date: Tue Feb 18 19:49:22 2014 -0800
summary:
Major updates for my local 3.4 build tool.
files:
3.4/threefourtool.py | 346 +++++++++++++++++++++---------
1 files changed, 241 insertions(+), 105 deletions(-)
diff --git a/3.4/threefourtool.py b/3.4/threefourtool.py
--- a/3.4/threefourtool.py
+++ b/3.4/threefourtool.py
@@ -4,15 +4,33 @@
import collections
import datetime
from dryparse import dryparse
+import glob
import os
import pprint
import pickle
+import shutil
import subprocess
import sys
import tempfile
import time
+outgoing = "/home/larry/src/python/34outgoing"
+
+
+def yes_no():
+ while True:
+ s = input("Are you sure? (y/n) >")
+ if s.strip() in 'yn':
+ return s
+
+
+def now():
+ s = str(datetime.datetime.now())
+ for c in "- :":
+ s = s.replace(c, '.')
+ return s
+
def line_to_rev(line):
if line.startswith("changeset:"):
line = line[len("changeset:"):].strip()
@@ -24,6 +42,7 @@
# print(repr(line), rev)
return local, rev
+
changesets = collections.OrderedDict()
user_date_to_revs = {} # "user date" == "{user} {date}", maps to (rev, branch)
default_to_34 = {} # maps rev in default to rev in 3.4
@@ -31,10 +50,18 @@
revs = []
branches = {}
+def reset_changesets():
+ changesets.clear()
+ user_date_to_revs.clear()
+ default_to_34.clear()
+ default_from_34.clear()
+ revs.clear()
+ branches.clear()
+
def line_iterator(pipe, encoding='utf-8'):
text = b''
while True:
- got = pipe.read(4096)
+ got = pipe.read(128)
if not got:
break
# print("GOT", repr(got))
@@ -73,10 +100,12 @@
def get_user_date_to_revs(fields):
return user_date_to_revs[fields['user'] + ' ' + fields['date']]
-def read_changesets(earliest = '6343bdbb7085'):
- if changesets:
+def read_changesets(earliest = '6343bdbb7085', force=False):
+ if changesets and not force:
return
+ reset_changesets()
+
p = None
current_directory = os.getcwd()
try:
@@ -133,6 +162,7 @@
revs.append(rev)
if not branch in branches:
branches[branch] = []
+ assert rev not in branches[branch], "Adding rev " + rev + " twice to branch " + repr(branch) + "!"
branches[branch].append(rev)
if rev == earliest:
break
@@ -150,7 +180,7 @@
default_from_34[d['3.4']] = d[None]
def header(category, printer):
- print(category)
+ # print(category)
printer("""
<h2>{}</h2>
<table border=0 cellpadding=5px>
@@ -161,7 +191,6 @@
</td></tr></table>
""")
-
def print_rev(rev, printer):
fields = changesets[rev]
d = dict(fields)
@@ -169,7 +198,7 @@
d['rev'] = rev
d['description0'] = description[0]
d['description1+'] = "<br/>\n".join(description[1:])
- print(" r", rev)
+ # print(" r", rev)
printer("""
<tr>
@@ -200,10 +229,12 @@
seen = set()
for rev, fields in changesets.items():
- print(" :", rev)
+ # print(" :", rev)
rev_to_print = print_test(rev)
if rev_to_print:
break
+ else:
+ return
print_rev(rev_to_print, printer)
seen.add(rev_to_print)
@@ -214,21 +245,21 @@
hopper.sort()
parents = fields['parents']
- print(" r", rev, "p", parents)
+ # print(" r", rev, "p", parents)
refill_hopper(parents)
while hopper:
_, rev = hopper.pop(0)
- print(" ?", rev)
+ # print(" ?", rev)
if not lineage_test(rev):
continue
fields = changesets[rev]
parents = fields['parents']
- print(" r", rev, "p", parents)
refill_hopper(parents)
rev_to_print = print_test(rev)
if not rev_to_print:
continue
+ # print(" r", rev, "p", parents, "rev to print", rev_to_print)
if rev_to_print in seen:
continue
seen.add(rev_to_print)
@@ -252,17 +283,17 @@
return rev
def is_34(rev):
- # if rev == '6343bdbb7085':
- # raise StopIteration()
fields = changesets[rev]
branch = fields.get('branch')
if branch != '3.4':
return False
revs = get_user_date_to_revs(fields)
assert (rev, branch) in revs
- for rev, branch in revs:
+ # print("is 34?", rev, revs)
+ for r2, branch in revs:
if not branch:
- return rev
+ # print(rev, "->", r2)
+ return r2
return False
class Tool:
@@ -296,7 +327,7 @@
"""
Regenerate the status webpage.
"""
- f = open("/home/larry/src/python/3.4.merge.status.html", "wt")
+ f = open(outgoing + "/merge.status.html", "wt")
read_changesets()
@@ -328,7 +359,7 @@
</font></p>
""")
- printer("<div style='background-color:#ffc0c0;'><font size=7 color=#800000>Notice: this is <b>BETA</b> (don't take it seriously yet)</font></div>")
+ # printer("<div style='background-color:#ffc0c0;'><font size=7 color=#800000>Notice: this is <b>BETA</b> (don't take it seriously yet)</font></div>")
header("Merged", printer)
print_revs(is_34, is_34, printer)
footer(printer)
@@ -350,20 +381,12 @@
f.close()
- def _run_commands(self, commands, u):
+ def _run_command(self, commands, u):
commands_run = u['commands run']
- def done():
- self.unfinished = None
- self._abandon()
- sys.exit(0)
-
- done_tuple = ("Done!", done)
- if done_tuple not in commands:
- commands.append(done_tuple)
while True:
print("Commands:")
- commands_set = set('h.d')
+ commands_set = set('h.dq')
for i, (text, cmd) in enumerate(commands):
label = str(i+1)
commands_set.add(label)
@@ -387,6 +410,8 @@
pprint.pprint(u)
print()
continue
+ if s == 'q':
+ return
if s == 'h':
print("""
Simple help
@@ -394,6 +419,7 @@
. - repeat the command menu
h - print this help message
d - print the internal "unfinished" dict
+q - quit
""")
continue
break
@@ -409,74 +435,154 @@
else:
cmd()
- def pick(self, picked_revision):
+ def pick(self, picked_revision, *picked_revisions):
"""
Cherry-pick a revision from default to 3.4.
"""
+ pr = [picked_revision]
+ pr.extend(picked_revisions)
+ picked_revisions = pr
+
+ read_changesets()
+
+ # sort earliest first
+ for rev in picked_revisions:
+ if rev not in branches[None]:
+ sys.exit("Can't pick revision " + rev + ", it's not in default.")
+
+ def to_default_index(rev):
+ # revisions are sorted latest first,
+ # we want the opposite, so we negate
+ return -branches[None].index(rev)
+ picked_revisions.sort(key=to_default_index)
+
if self.unfinished:
- if self.unfinished['default picked revision'] == picked_revision:
+ if self.unfinished['original picked revisions'] == picked_revisions:
return self.finish()
sys.exit("You have unfinished business!\n\nUse the 'finish' command to finish it,\nor the 'abandon' command to abandon it.")
- print("Picking revision", repr(picked_revision))
- read_changesets()
- # changesets = collections.OrderedDict()
- # user_date_to_revs = {} # "user date" == "{user} {date}", maps to (rev, branch)
- # revs = []
- # branches = {}
- try:
- index = branches[None].index(picked_revision)
- except ValueError:
- sys.exit("{} is not a revision in Python trunk.".format(picked_revision))
- r_34_head = branches['3.4'][0]
- r_34_first_revision = branches['3.4'][-1]
- # print("index", index)
- # print("3.4 branch", branches['3.4'])
- # print("default_from_34", default_from_34)
- r_previous = None
- # find where it should go in 3.4
- for r in branches['3.4']:
- # print("r", r)
- if r == r_34_first_revision:
- break
- r_default = default_from_34[r]
- i = branches[None].index(r_default)
- if i == index:
- sys.exit("{} already in 3.4!".format(picked_revision))
- if i > index:
- break
- r_previous = r
- else:
- sys.exit("Unexpected non-termination in pick!")
-
- if r == r_34_head:
- r_rebase_from = None
- else:
- r_rebase_from = r_previous
-
- # figure out previous revision in default, in case we need to make a patch
- diff_from = branches[None][index + 1]
-
- fields = changesets[picked_revision]
-
self.unfinished = {
'function': '_pick',
- 'threefour rebase from': r_rebase_from,
- 'threefour graft here': r,
- 'default picked revision': picked_revision,
- 'user': fields['user'],
- 'date': fields['date'],
- 'description': '\n'.join(fields['description']),
- 'default diff from': diff_from,
- 'threefour picked revision': 'UNKNOWN (detect via "Detect new revision")',
- 'commands run': set(),
+ 'picked revisions': picked_revisions,
+ 'original picked revisions': list(picked_revisions),
}
self.finish()
+
+ def _analyze_picked_revision(self):
+ read_changesets(force=True)
+ picked_revisions = self.unfinished['picked revisions']
+ # print("picked_revisions", picked_revisions)
+ while picked_revisions:
+ # changesets = collections.OrderedDict()
+ # user_date_to_revs = {} # "user date" == "{user} {date}", maps to (rev, branch)
+ # revs = []
+ # branches = {}
+ picked_revision = picked_revisions[0]
+ try:
+ index = branches[None].index(picked_revision)
+ except ValueError:
+ sys.exit("{} is not a revision in Python trunk.".format(picked_revision))
+ r_34_head = branches['3.4'][0]
+ r_34_first_revision = branches['3.4'][-1]
+ # print("index", index)
+ # print("3.4 branch", branches['3.4'])
+ # print("default_from_34", default_from_34)
+ r_previous = None
+ # find where it should go in 3.4
+ i = None
+ for r in branches['3.4']:
+ # print("r", r)
+ if r == r_34_first_revision:
+ break
+ r_default = default_from_34[r]
+ i = branches[None].index(r_default)
+ if i >= index:
+ break
+ r_previous = r
+ else:
+ sys.exit("Unexpected non-termination when searching for right spot to graft to!")
+
+ if i == index:
+ print("{} already in 3.4!".format(picked_revision))
+ picked_revisions.pop(0)
+ continue
+
+ if r == r_34_head:
+ r_rebase_from = None
+ else:
+ r_rebase_from = r_previous
+
+ # print("chose r", r, "r_34_head", r_34_head, "r_rebase_from", r_rebase_from)
+
+
+ # figure out previous revision in default, in case we need to make a patch
+ diff_from = branches[None][index + 1]
+
+ fields = changesets[picked_revision]
+
+ u2 = dict(self.unfinished)
+
+ if 'done' in u2:
+ del u2['done']
+
+ u2.update({
+ 'threefour rebase from': r_rebase_from,
+ 'threefour graft here': r,
+ 'default picked revision': picked_revision,
+ 'user': fields['user'],
+ 'date': fields['date'],
+ 'description': '\n'.join(fields['description']),
+ 'default diff from': diff_from,
+ 'threefour picked revision': 'UNKNOWN (detect via "Detect new revision")',
+ 'commands run': set(),
+ })
+ picked_revisions.pop(0)
+ self.unfinished = u2
+ break
+
def _pick(self):
+ read_changesets()
os.chdir("/home/larry/src/python/3.4")
+ print("Updating to 3.4 branch:")
+ os.system("hg update -r 3.4")
+ print()
+ u = self.unfinished.get
+ print("Picking revisions ", u('picked revisions'))
+ while True:
+ picked_revisions = u('picked revisions')
+ if not picked_revisions:
+ break
+ if 'default picked revision' not in self.unfinished:
+ self._analyze_picked_revision()
+ if 'default picked revision' in self.unfinished:
+ self._pick_revision()
+ if not u('picked revisions'):
+ self._abandon()
+
+ def _pick_revision(self):
u = self.unfinished
+ u['EDITOR'] = os.getenv('EDITOR')
+
+ patch_path = "/tmp/patch.{default diff from}.to.{default picked revision}.diff".format_map(u)
+ u['patch path'] = patch_path
+
+ f, commit_message_path = tempfile.mkstemp(suffix='txt')
+ os.close(f)
+ with open(commit_message_path, 'wt') as f:
+ f.write(u['description'])
+ u['commit message path'] = commit_message_path
+
+ def delete_files(*a):
+ for path in a:
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+ atexit.register(delete_files, patch_path, commit_message_path)
+
def detect_new_revision():
output = subprocess.check_output(['/usr/bin/hg', 'summary']).decode('utf-8').split('\n')
line = output[0]
@@ -487,26 +593,10 @@
r = r.split()[0].strip()
u['threefour picked revision'] = r
- while True:
- patch_path = "/tmp/patch.{default diff from}.to.{default picked revision}.diff".format_map(u)
- u['patch path'] = patch_path
+ def mark_as_picked():
+ del self.unfinished['default picked revision']
- f, commit_message_path = tempfile.mkstemp(suffix='txt')
- os.close(f)
- with open(commit_message_path, 'wt') as f:
- f.write(u['description'])
- u['commit message path'] = commit_message_path
-
- def delete_files(*a):
- for path in a:
- try:
- os.unlink(path)
- except OSError:
- pass
-
- atexit.register(delete_files, patch_path, commit_message_path)
-
- u['EDITOR'] = os.getenv('EDITOR')
+ while u.get('default picked revision'):
commands = []
commands.append(("Update to appropriate revision in 3.4 branch", "hg update -r {threefour graft here}"))
@@ -521,10 +611,13 @@
commands.append(("Detect new revision", detect_new_revision))
c = "hg rebase --source {threefour rebase from} --dest {threefour picked revision}"
commands.append(("Rebase subsequent revisions after grafted revision", c))
+ commands.append(("Update to head of 3.4 branch", "hg update -r 3.4"))
- commands.append(("Update to head of 3.4 branch", "hg update -r 3.4"))
+ commands.append(("Mark revision as picked", mark_as_picked))
- self._run_commands(commands, u)
+ print()
+ print("Picking revision {default picked revision}:".format_map(u))
+ self._run_command(commands, u)
def finish(self):
try:
@@ -540,16 +633,59 @@
except KeyboardInterrupt:
self._save()
- def abandon(self):
+ def abandon(self, *, force:('-f',)=False):
if not self.unfinished:
sys.exit("No unfinished business!")
- while True:
- s = input("Are you sure? (y/n) >")
- if s.strip() in 'yn':
- break
- if s == 'y':
+ if force or yes_no() == 'y':
self._abandon()
+ def recreate(self, *, force:('-f',)=False):
+ """
+ Recreate the 3.4 branch from scratch.
+ """
+ if (not force) and (yes_no() == 'n'):
+ sys.exit()
+ os.chdir("/home/larry/src/python")
+ while True:
+ if os.path.isdir("/home/larry/src/python/bad3.4"):
+ shutil.rmtree("/home/larry/src/python/bad3.4")
+ continue
+ if os.path.isdir("/home/larry/src/python/3.4"):
+ os.rename("/home/larry/src/python/3.4", "/home/larry/src/python/bad3.4")
+ continue
+ break
+ os.system("hg clone trunk 3.4")
+ os.chdir("/home/larry/src/python/3.4")
+ os.system("hg update -r e64ae8b82672")
+ os.system("hg branch 3.4")
+ os.system("hg commit -m 'Created release branch for 3.4.'")
+
+ def tar(self):
+ time = now()
+ tardir = "/home/larry/src/python/python-" + time
+ def remove_dir(dir):
+ if os.path.isdir(dir):
+ shutil.rmtree(dir)
+ if os.path.isdir(dir):
+ sys.exit("Couldn't remove directory" + repr(dir))
+ remove_dir(tardir)
+
+ os.chdir("/home/larry/src/python")
+ os.system("hg clone 3.4 " + tardir)
+
+ os.chdir(tardir)
+ remove_dir(".hg")
+ for prefix in ('.hg', '.bzr', '.git'):
+ for filename in glob.glob(prefix + '*'):
+ os.unlink(filename)
+ os.chdir("/home/larry/src/python")
+ os.system("tar cvfz " + outgoing + "/python.3.4.{}.tgz python-{}".format(time, time))
+
+ remove_dir(tardir)
+
+ def rsync(self):
+ os.chdir(outgoing)
+ os.system("rsync -av * midwinter.com:public_html/3.4.status")
--
Repository URL: http://hg.python.org/release
More information about the Python-checkins
mailing list