From 3f4291bb18668d9ca2ec9319e2ab44940cf2a3f5 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 24 Feb 2013 11:37:13 +0100 Subject: [PATCH 001/132] Handle off-by-1 errors in cdparanoia progress parsing Fixes the exception I got on ripping The Strokes - Someday --- HACKING | 3 + morituri/common/common.py | 2 +- morituri/program/cdparanoia.py | 28 +++--- morituri/test/Makefile.am | 2 + morituri/test/cdparanoia.progress.strokes | 111 ++++++++++++++++++++++ morituri/test/strokes-someday.toc | 12 +++ morituri/test/test_image_toc.py | 18 ++++ morituri/test/test_program_cdparanoia.py | 18 +++- 8 files changed, 181 insertions(+), 13 deletions(-) create mode 100644 morituri/test/cdparanoia.progress.strokes create mode 100644 morituri/test/strokes-someday.toc diff --git a/HACKING b/HACKING index 44723c3..cc25424 100644 --- a/HACKING +++ b/HACKING @@ -41,3 +41,6 @@ CDROMS PLEXTOR CD-R PX-W8432T Read offset of device is: 355. +Test discs +---------- +The Strokes - Someday (promo): has 1 frame silence marked as SILENCE diff --git a/morituri/common/common.py b/morituri/common/common.py index a647ba0..0d91ed2 100644 --- a/morituri/common/common.py +++ b/morituri/common/common.py @@ -30,7 +30,7 @@ from morituri.extern.log import log FRAMES_PER_SECOND = 75 -SAMPLES_PER_FRAME = 588 +SAMPLES_PER_FRAME = 588 # a sample is 2 16-bit values, left and right channel WORDS_PER_FRAME = SAMPLES_PER_FRAME * 2 BYTES_PER_FRAME = SAMPLES_PER_FRAME * 4 diff --git a/morituri/program/cdparanoia.py b/morituri/program/cdparanoia.py index 406d6d9..e8f6c59 100644 --- a/morituri/program/cdparanoia.py +++ b/morituri/program/cdparanoia.py @@ -64,19 +64,21 @@ class ChecksumException(Exception): pass +# example: +# ##: 0 [read] @ 24696 _PROGRESS_RE = re.compile(r""" - ^\#\#: (?P.+)\s # function code - \[(?P.*)\]\s@\s # function name - (?P\d+) # offset + ^\#\#: (?P.+)\s # function code + \[(?P.*)\]\s@\s # [function name] @ + (?P\d+) # offset in words (2-byte one channel value) """, re.VERBOSE) _ERROR_RE = re.compile("^scsi_read error:") # from reading cdparanoia source code, it looks like offset is reported in -# number of single-channel samples, ie. 2 bytes per unit, and absolute +# number of single-channel samples, ie. 2 bytes (word) per unit, and absolute -class ProgressParser(object): +class ProgressParser(log.Loggable): read = 0 # last [read] frame wrote = 0 # last [wrote] frame errors = 0 # count of number of scsi errors @@ -128,13 +130,15 @@ class ProgressParser(object): # set nframes if not yet set if self._nframes is None and self.read != 0: self._nframes = frameOffset - self.read + self.debug('set nframes to %r', self._nframes) # set firstFrames if not yet set if self._firstFrames is None: self._firstFrames = frameOffset - self.start + self.debug('set firstFrames to %r', self._firstFrames) markStart = None - markEnd = None + markEnd = None # the next unread frame (half-inclusive) # verify it either read nframes more or went back for verify if frameOffset > self.read: @@ -165,10 +169,11 @@ class ProgressParser(object): # cdparanoia reads quite a bit beyond the current track before it # goes back to verify; don't count those - if markEnd > self.stop: - markEnd = self.stop - if markStart > self.stop: - markStart = self.stop + # markStart, markEnd of 0, 21 with stop 0 should give 1 read + if markEnd > self.stop + 1: + markEnd = self.stop + 1 + if markStart > self.stop + 1: + markStart = self.stop + 1 self.reads += markEnd - markStart @@ -185,8 +190,9 @@ class ProgressParser(object): Each frame gets read twice. More than two reads for a frame reduce track quality. """ - frames = self.stop - self.start + 1 + frames = self.stop - self.start + 1 # + 1 since stop is inclusive reads = self.reads + self.debug('getTrackQuality: frames %d, reads %d' % (frames, reads)) # don't go over a 100%; we know cdparanoia reads each frame at least # twice diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index e9f0751..298ada2 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -44,8 +44,10 @@ EXTRA_DIST = \ track-single.cue \ cdparanoia.progress \ cdparanoia.progress.error \ + cdparanoia.progress.strokes \ cdrdao.readtoc.progress \ silentalarm.result.pickle \ + strokes-someday.toc \ totbl.fast.toc \ track.flac \ cache/result/fe105a11.pickle \ diff --git a/morituri/test/cdparanoia.progress.strokes b/morituri/test/cdparanoia.progress.strokes new file mode 100644 index 0000000..3369ab5 --- /dev/null +++ b/morituri/test/cdparanoia.progress.strokes @@ -0,0 +1,111 @@ +Sending all callbacks to stderr for wrapper script +cdparanoia III release 10.2 (September 11, 2008) + +Ripping from sector 0 (track 0 [0:00.00]) + to sector 0 (track 0 [0:00.00]) + +outputting to cdda.wav + +##: 0 [read] @ 24696 +##: 0 [read] @ 56448 +##: 0 [read] @ 88200 +##: 0 [read] @ 119952 +##: 0 [read] @ 151704 +##: 0 [read] @ 183456 +##: 0 [read] @ 215208 +##: 0 [read] @ 246960 +##: 0 [read] @ 278712 +##: 0 [read] @ 310464 +##: 0 [read] @ 342216 +##: 0 [read] @ 373968 +##: 0 [read] @ 405720 +##: 0 [read] @ 437472 +##: 0 [read] @ 469224 +##: 0 [read] @ 500976 +##: 0 [read] @ 532728 +##: 0 [read] @ 564480 +##: 0 [read] @ 596232 +##: 0 [read] @ 627984 +##: 0 [read] @ 659736 +##: 0 [read] @ 691488 +##: 0 [read] @ 723240 +##: 0 [read] @ 754992 +##: 0 [read] @ 786744 +##: 0 [read] @ 818496 +##: 0 [read] @ 850248 +##: 0 [read] @ 882000 +##: 0 [read] @ 913752 +##: 0 [read] @ 945504 +##: 0 [read] @ 977256 +##: 0 [read] @ 1009008 +##: 0 [read] @ 1040760 +##: 0 [read] @ 1072512 +##: 0 [read] @ 1104264 +##: 0 [read] @ 1136016 +##: 0 [read] @ 1167768 +##: 0 [read] @ 1199520 +##: 0 [read] @ 1231272 +##: 0 [read] @ 1263024 +##: 0 [read] @ 1294776 +##: 0 [read] @ 1326528 +##: 0 [read] @ 1358280 +##: 0 [read] @ 1390032 +##: 0 [read] @ 1410024 +##: 0 [read] @ 23520 +##: 0 [read] @ 55272 +##: 0 [read] @ 87024 +##: 0 [read] @ 118776 +##: 0 [read] @ 150528 +##: 0 [read] @ 182280 +##: 0 [read] @ 214032 +##: 0 [read] @ 245784 +##: 0 [read] @ 277536 +##: 0 [read] @ 309288 +##: 0 [read] @ 341040 +##: 0 [read] @ 372792 +##: 0 [read] @ 404544 +##: 0 [read] @ 436296 +##: 0 [read] @ 468048 +##: 0 [read] @ 499800 +##: 0 [read] @ 531552 +##: 0 [read] @ 563304 +##: 0 [read] @ 595056 +##: 0 [read] @ 626808 +##: 0 [read] @ 658560 +##: 0 [read] @ 690312 +##: 0 [read] @ 722064 +##: 0 [read] @ 753816 +##: 0 [read] @ 785568 +##: 0 [read] @ 817320 +##: 0 [read] @ 849072 +##: 0 [read] @ 880824 +##: 0 [read] @ 912576 +##: 0 [read] @ 944328 +##: 0 [read] @ 976080 +##: 0 [read] @ 1007832 +##: 0 [read] @ 1039584 +##: 0 [read] @ 1071336 +##: 0 [read] @ 1103088 +##: 0 [read] @ 1134840 +##: 0 [read] @ 1166592 +##: 0 [read] @ 1198344 +##: 0 [read] @ 1230096 +##: 0 [read] @ 1261848 +##: 0 [read] @ 1293600 +##: 0 [read] @ 1325352 +##: 0 [read] @ 1357104 +##: 0 [read] @ 1388856 +##: 0 [read] @ 1410024 +##: 1 [verify] @ 0 +##: 3 [correction] @ 1005459 +##: 3 [correction] @ 1005480 +##: 1 [verify] @ 1005480 +##: 1 [verify] @ 1005480 +##: -2 [wrote] @ 1175 +##: -2 [wrote] @ 1176 +##: -1 [finished] @ 1175 + + +Done. + + diff --git a/morituri/test/strokes-someday.toc b/morituri/test/strokes-someday.toc new file mode 100644 index 0000000..bafd8e0 --- /dev/null +++ b/morituri/test/strokes-someday.toc @@ -0,0 +1,12 @@ +CD_DA + + +// Track 1 +TRACK AUDIO +COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +SILENCE 00:00:01 +FILE "data.wav" 0 03:06:59 +START 00:00:01 + diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 1955d03..0175bed 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -318,3 +318,21 @@ class TOTBLTestCase(common.TestCase): def testCDDBId(self): self.toc.table.absolutize() self.assertEquals(self.toc.table.getCDDBDiscId(), '810b7b0b') + + +# The Strokes - Someday has a 1 frame SILENCE marked as such in toc + + +class StrokesTestCase(common.TestCase): + + def setUp(self): + self.path = os.path.join(os.path.dirname(__file__), + u'strokes-someday.toc') + self.toc = toc.TocFile(self.path) + self.toc.parse() + self.assertEquals(len(self.toc.table.tracks), 1) + + def testIndexes(self): + t = self.toc.table.tracks[0] + self.assertEquals(t.getIndex(0).relative, 0) + self.assertEquals(t.getIndex(1).relative, 1) diff --git a/morituri/test/test_program_cdparanoia.py b/morituri/test/test_program_cdparanoia.py index 58e0bbc..397073d 100644 --- a/morituri/test/test_program_cdparanoia.py +++ b/morituri/test/test_program_cdparanoia.py @@ -25,7 +25,23 @@ class ParseTestCase(common.TestCase): self._parser.parse(line) q = '%.01f %%' % (self._parser.getTrackQuality() * 100.0, ) - self.assertEquals(q, '99.7 %') + self.assertEquals(q, '99.6 %') + +class Parse1FrameTestCase(common.TestCase): + + def setUp(self): + path = os.path.join(os.path.dirname(__file__), + 'cdparanoia.progress.strokes') + self._parser = cdparanoia.ProgressParser(start=0, stop=0) + + self._handle = open(path) + + def testParse(self): + for line in self._handle.readlines(): + self._parser.parse(line) + + q = '%.01f %%' % (self._parser.getTrackQuality() * 100.0, ) + self.assertEquals(q, '100.0 %') class ErrorTestCase(common.TestCase): From d882f845ffa5025e741ab91fcb36e5055a2dd568 Mon Sep 17 00:00:00 2001 From: Nicolas Cornu Date: Tue, 26 Feb 2013 21:17:38 +0100 Subject: [PATCH 002/132] Use os.path.join instead of hardcoded paths --- morituri/common/directory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/morituri/common/directory.py b/morituri/common/directory.py index 66fb510..e076a1f 100644 --- a/morituri/common/directory.py +++ b/morituri/common/directory.py @@ -34,7 +34,7 @@ class Directory(log.Loggable): path = os.path.join(directory, 'morituri.conf') self.info('Using XDG, configuration file is %s' % path) except ImportError: - path = os.path.expanduser('~/.moriturirc') + path = os.path.join(os.path.expanduser('~'), '.moriturirc') self.info('Not using XDG, configuration file is %s' % path) return path @@ -45,7 +45,7 @@ class Directory(log.Loggable): path = BaseDirectory.save_cache_path('morituri') self.info('Using XDG, cache directory is %s' % path) except ImportError: - path = os.path.expanduser('~/.morituri/cache') + path = os.path.join(os.path.expanduser('~'), '.morituri', 'cache') if not os.path.exists(path): os.makedirs(path) self.info('Not using XDG, cache directory is %s' % path) @@ -68,7 +68,7 @@ class Directory(log.Loggable): except ImportError: pass - path = os.path.expanduser('~/.morituri/cache') + path = os.path.join(os.path.expanduser('~'), '.morituri', 'cache') if os.path.exists(path): self.info('From before XDG, read cache directory is %s' % path) paths.append(path) From 45725999d255abc25aec401343a042463e6d5b2b Mon Sep 17 00:00:00 2001 From: Nicolas Cornu Date: Tue, 26 Feb 2013 21:28:19 +0100 Subject: [PATCH 003/132] Use with statement to open files --- morituri/common/renamer.py | 60 ++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/morituri/common/renamer.py b/morituri/common/renamer.py index b92f1f4..0373064 100644 --- a/morituri/common/renamer.py +++ b/morituri/common/renamer.py @@ -49,24 +49,23 @@ class Operator(object): Verifies the state. """ todo = os.path.join(self._statePath, self._key + '.todo') - handle = open(todo, 'r') lines = [] - for line in handle.readlines(): - lines.append(line) - name, data = line.split(' ', 1) - cls = globals()[name] - operation = cls.deserialize(data) - self._todo.append(operation) + with open(todo, 'r') as handle: + for line in handle.readlines(): + lines.append(line) + name, data = line.split(' ', 1) + cls = globals()[name] + operation = cls.deserialize(data) + self._todo.append(operation) done = os.path.join(self._statePath, self._key + '.done') - i = 0 if os.path.exists(done): - handle = open(done, 'r') - for i, line in enumerate(handle.readlines()): - assert line == lines[i], "line %s is different than %s" % ( - line, lines[i]) - self._done.append(self._todo[i]) + with open(done, 'r') as handle: + for i, line in enumerate(handle.readlines()): + assert line == lines[i], "line %s is different than %s" % ( + line, lines[i]) + self._done.append(self._todo[i]) # last task done is i; check if the next one might have gotten done. self._resuming = True @@ -78,21 +77,19 @@ class Operator(object): # only save todo first time todo = os.path.join(self._statePath, self._key + '.todo') if not os.path.exists(todo): - handle = open(todo, 'w') - for o in self._todo: - name = o.__class__.__name__ - data = o.serialize() - handle.write('%s %s\n' % (name, data)) - handle.close() + with open(todo, 'w') as handle: + for o in self._todo: + name = o.__class__.__name__ + data = o.serialize() + handle.write('%s %s\n' % (name, data)) # save done every time done = os.path.join(self._statePath, self._key + '.done') - handle = open(done, 'w') - for o in self._done: - name = o.__class__.__name__ - data = o.serialize() - handle.write('%s %s\n' % (name, data)) - handle.close() + with open(done, 'w') as handle: + for o in self._done: + name = o.__class__.__name__ + data = o.serialize() + handle.write('%s %s\n' % (name, data)) def start(self): """ @@ -203,15 +200,14 @@ class RenameInFile(Operation): # check if the source exists in the given file def do(self): - handle = open(self._path) - (fd, name) = tempfile.mkstemp(suffix='.morituri') + with open(self._path) as handle: + (fd, name) = tempfile.mkstemp(suffix='.morituri') - for s in handle: - os.write(fd, s.replace(self._source, self._destination)) + for s in handle: + os.write(fd, s.replace(self._source, self._destination)) - handle.close() - os.close(fd) - os.rename(name, self._path) + os.close(fd) + os.rename(name, self._path) def serialize(self): return '"%s" "%s" "%s"' % (self._path, self._source, self._destination) From f05ec0f2e97f0fa09cc1556b3db5f9850926b8d8 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 24 Feb 2013 12:19:21 +0100 Subject: [PATCH 004/132] The logic was the wrong way around. The idea is that we set an interval that is definitely smaller than the whole file. --- morituri/common/encode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/morituri/common/encode.py b/morituri/common/encode.py index dba0eb1..7b0ad8d 100644 --- a/morituri/common/encode.py +++ b/morituri/common/encode.py @@ -239,8 +239,8 @@ class EncodeTask(ctask.GstPipelineTask): # set an interval that is smaller than the duration # FIXME: check level and make sure it emits level up to the last # sample, even if input is small - interval = 1000000000L - if interval < duration: + interval = self.gst.SECOND + if interval > duration: interval = duration / 2 self._level.set_property('interval', interval) # add a probe so we can track progress From a6925423073a8b13a8237a016873f75f823530b0 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 24 Feb 2013 12:19:46 +0100 Subject: [PATCH 005/132] debug and cleanup --- morituri/common/encode.py | 14 ++++++++++---- morituri/rip/debug.py | 5 ++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/morituri/common/encode.py b/morituri/common/encode.py index 7b0ad8d..1fbd2a1 100644 --- a/morituri/common/encode.py +++ b/morituri/common/encode.py @@ -242,6 +242,8 @@ class EncodeTask(ctask.GstPipelineTask): interval = self.gst.SECOND if interval > duration: interval = duration / 2 + self.debug('Setting level interval to %s, duration %s', + self.gst.TIME_ARGS(interval), self.gst.TIME_ARGS(duration)) self._level.set_property('interval', interval) # add a probe so we can track progress # we connect to level because this gives us offset in samples @@ -291,10 +293,14 @@ class EncodeTask(ctask.GstPipelineTask): if self._peakdB is not None: self.debug('peakdB %r', self._peakdB) self.peak = math.sqrt(math.pow(10, self._peakdB / 10.0)) - else: - self.warning('No peak found, something went wrong!') - # workaround for when the file is too short to have volume ? - # self.peak = 0.0 + return + + self.warning('No peak found.') + + if self._duration: + self.warning('GStreamer level element did not send messages.') + # workaround for when the file is too short to have volume ? + # self.peak = 0.0 class TagReadTask(ctask.GstPipelineTask): diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index 5bb6a68..e7f09d0 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -49,7 +49,7 @@ class RCList(logcommand.LogCommand): self.stdout.write('%s: %s - %s\n' % ( cddbid, artist.encode('utf-8'), title.encode('utf-8'))) - + class RCLog(logcommand.LogCommand): @@ -155,6 +155,9 @@ class Encode(logcommand.LogCommand): runner.run(encodetask) + self.stdout.write('Peak level: %r\n' % encodetask.peak) + self.stdout.write('Encoded to %s\n' % toPath.encode('utf-8')) + class Tag(logcommand.LogCommand): summary = "run a tag reading task" From f0528965059f50ebab47c503e49f99903a41685a Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 26 Feb 2013 23:11:24 +0100 Subject: [PATCH 006/132] change debug message --- morituri/common/gstreamer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/common/gstreamer.py b/morituri/common/gstreamer.py index f1a2a56..fd5c38a 100644 --- a/morituri/common/gstreamer.py +++ b/morituri/common/gstreamer.py @@ -42,7 +42,7 @@ def removeAudioParsers(): plugin = registry.find_plugin("audioparsers") if plugin: - log.debug('gstreamer', 'Found audioparsers plugin from %s %s', + log.debug('gstreamer', 'removing audioparsers plugin from %s %s', plugin.get_source(), plugin.get_version()) # the query bug was fixed after 0.10.30 and before 0.10.31 From 4e03a6e966e2cdff494457efa44ae368f86e016b Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 26 Feb 2013 23:11:42 +0100 Subject: [PATCH 007/132] work around GStreamer bug in flacdec for really short files --- morituri/common/checksum.py | 70 +++++++++++++++++++++++-------------- morituri/common/encode.py | 7 ++-- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/morituri/common/checksum.py b/morituri/common/checksum.py index 68bf524..dea979f 100644 --- a/morituri/common/checksum.py +++ b/morituri/common/checksum.py @@ -101,49 +101,65 @@ class ChecksumTask(log.Loggable, gstreamer.GstPipelineTask): appsink name=sink sync=False emit-signals=True ''' % gstreamer.quoteParse(self._path).encode('utf-8') + def _getSampleLength(self): + # get length in samples of file + sink = self.pipeline.get_by_name('sink') + + self.debug('query duration') + try: + length, qformat = sink.query_duration(gst.FORMAT_DEFAULT) + except gst.QueryError, e: + self.setException(e) + return None + + # wavparse 0.10.14 returns in bytes + if qformat == gst.FORMAT_BYTES: + self.debug('query returned in BYTES format') + length /= 4 + self.debug('total sample length of file: %r', length) + + return length + + def paused(self): sink = self.pipeline.get_by_name('sink') - if self._sampleLength < 0: - self.debug('query duration') - try: - length, qformat = sink.query_duration(gst.FORMAT_DEFAULT) - except gst.QueryError, e: - self.setException(e) - return + length = self._getSampleLength() + if length is None: + return - # wavparse 0.10.14 returns in bytes - if qformat == gst.FORMAT_BYTES: - self.debug('query returned in BYTES format') - length /= 4 - self.debug('total sample length of file: %r', length) + if self._sampleLength < 0: self._sampleLength = length - self._sampleStart self.debug('sampleLength is queried as %d samples', self._sampleLength) else: self.debug('sampleLength is known, and is %d samples' % self._sampleLength) + self._sampleEnd = self._sampleStart + self._sampleLength - 1 self.debug('sampleEnd is sample %d' % self._sampleEnd) self.debug('event') - # the segment end only is respected since -good 0.10.14.1 - event = gst.event_new_seek(1.0, gst.FORMAT_DEFAULT, - gst.SEEK_FLAG_FLUSH, - gst.SEEK_TYPE_SET, self._sampleStart, - gst.SEEK_TYPE_SET, self._sampleEnd + 1) # half-inclusive interval - self.debug('CRCing %r from sector %d to sector %d' % ( - self._path, - self._sampleStart / common.SAMPLES_PER_FRAME, - (self._sampleEnd + 1) / common.SAMPLES_PER_FRAME)) - # FIXME: sending it with sampleEnd set screws up the seek, we don't get - # everything for flac; fixed in recent -good - result = sink.send_event(event) - self.debug('event sent, result %r', result) - if not result: - self.error('Failed to select samples with GStreamer seek event') + if self._sampleStart == 0 and self._sampleEnd + 1 == length: + self.debug('No need to seek, crcing full file') + else: + # the segment end only is respected since -good 0.10.14.1 + event = gst.event_new_seek(1.0, gst.FORMAT_DEFAULT, + gst.SEEK_FLAG_FLUSH, + gst.SEEK_TYPE_SET, self._sampleStart, + gst.SEEK_TYPE_SET, self._sampleEnd + 1) # half-inclusive + self.debug('CRCing %r from frame %d to frame %d (excluded)' % ( + self._path, + self._sampleStart / common.SAMPLES_PER_FRAME, + (self._sampleEnd + 1) / common.SAMPLES_PER_FRAME)) + # FIXME: sending it with sampleEnd set screws up the seek, we + # don't get # everything for flac; fixed in recent -good + result = sink.send_event(event) + self.debug('event sent, result %r', result) + if not result: + self.error('Failed to select samples with GStreamer seek event') sink.connect('new-buffer', self._new_buffer_cb) sink.connect('eos', self._eos_cb) diff --git a/morituri/common/encode.py b/morituri/common/encode.py index 1fbd2a1..4dfdc32 100644 --- a/morituri/common/encode.py +++ b/morituri/common/encode.py @@ -167,6 +167,7 @@ class EncodeTask(ctask.GstPipelineTask): self._inpath = inpath self._outpath = outpath self._taglist = taglist + self._length = 0 # in samples self._level = None self._peakdB = None @@ -299,8 +300,10 @@ class EncodeTask(ctask.GstPipelineTask): if self._duration: self.warning('GStreamer level element did not send messages.') - # workaround for when the file is too short to have volume ? - # self.peak = 0.0 + # workaround for when the file is too short to have volume ? + if self._length == common.SAMPLES_PER_FRAME: + self.warning('only one frame of audio, setting peak to 0.0') + self.peak = 0.0 class TagReadTask(ctask.GstPipelineTask): From 55a2325e1ecda99860e341d618bc06c9c642053e Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Fri, 1 Mar 2013 18:01:25 +0100 Subject: [PATCH 008/132] add debugging, comments and documentation --- morituri/image/table.py | 8 ++++++++ morituri/image/toc.py | 33 ++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/morituri/image/table.py b/morituri/image/table.py index a34717a..18e9efb 100644 --- a/morituri/image/table.py +++ b/morituri/image/table.py @@ -95,6 +95,12 @@ class Track: return self.indexes[number] def getFirstIndex(self): + """ + Get the first chronological index for this track. + + Typically this is INDEX 01; but it could be INDEX 00 if there's + a pre-gap. + """ indexes = self.indexes.keys() indexes.sort() return self.indexes[indexes[0]] @@ -510,6 +516,8 @@ class Table(object, log.Loggable): Dump our internal representation to a .cue file content. + + @rtype: C{unicode} """ lines = [] diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 55f902d..9382dc2 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -22,6 +22,8 @@ """ Reading .toc files + +The .toc file format is described in the man page of cdrdao """ import re @@ -62,7 +64,7 @@ _FILE_RE = re.compile(r""" ^FILE # FILE \s+"(?P.*)" # 'file name' in quotes \s+(?P.+) # start offset - \s(?P.+)$ # stop offset + \s(?P.+)$ # length in frames of section """, re.VERBOSE) _DATAFILE_RE = re.compile(r""" @@ -106,14 +108,16 @@ class TocFile(object, log.Loggable): currentTrack = None state = 'HEADER' - counter = 0 + counter = 0 # counts sources for audio data; SILENCE/ZERO/FILE trackNumber = 0 indexNumber = 0 absoluteOffset = 0 # running absolute offset of where each track starts relativeOffset = 0 # running relative offset, relative to counter src - currentLength = 0 # accrued during TRACK record parsing, current track + currentLength = 0 # accrued during TRACK record parsing; + # length of current track as parsed so far; + # reset on each TRACK statement totalLength = 0 # accrued during TRACK record parsing, total disc - pregapLength = 0 # length of the pre-gap, current track + pregapLength = 0 # length of the pre-gap, current track in for loop # the first track's INDEX 1 can only be gotten from the .toc @@ -165,20 +169,26 @@ class TocFile(object, log.Loggable): absolute=absoluteOffset + pregapLength, relative=relativeOffset + pregapLength, counter=counter) - self.debug('track %d, added index %r', - currentTrack.number, currentTrack.getIndex(1)) + self.debug( + '[track %2d index 01] pregapLength %r, added %r', + currentTrack.number, pregapLength, + currentTrack.getIndex(1)) + # update running totals trackNumber += 1 absoluteOffset += currentLength relativeOffset += currentLength totalLength += currentLength + trackMode = m.group('mode') + + # reset counters currentLength = 0 indexNumber = 1 - trackMode = m.group('mode') pregapLength = 0 # FIXME: track mode - self.debug('found track %d, mode %s', trackNumber, trackMode) + self.debug('found track %d, mode %s, at absoluteOffset %d', + trackNumber, trackMode, absoluteOffset) audio = trackMode == 'AUDIO' currentTrack = table.Track(trackNumber, audio=audio) self.table.tracks.append(currentTrack) @@ -281,15 +291,16 @@ class TocFile(object, log.Loggable): offset = common.msfToFrames(m.group('offset')) currentTrack.index(indexNumber, path=currentFile.path, relative=offset, counter=counter) - self.debug('track %d, added index %r', - currentTrack.number, currentTrack.getIndex(indexNumber)) + self.debug('[track %2d index %2d] added %r', + currentTrack.number, indexNumber, + currentTrack.getIndex(indexNumber)) # handle index 1 of final track, if any if currentTrack: currentTrack.index(1, path=currentFile.path, absolute=absoluteOffset + pregapLength, relative=relativeOffset + pregapLength, counter=counter) - self.debug('track %d, added index %r', + self.debug('[track %2d index 01] last track, added %r', currentTrack.number, currentTrack.getIndex(1)) # totalLength was added up to the penultimate track From fdab6f1e3f0ae4ce3e7b373fff3aad861a88212d Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Fri, 1 Mar 2013 18:44:47 +0100 Subject: [PATCH 009/132] fix doc --- morituri/rip/image.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/morituri/rip/image.py b/morituri/rip/image.py index a9afef8..4c19c15 100644 --- a/morituri/rip/image.py +++ b/morituri/rip/image.py @@ -200,8 +200,13 @@ class Rename(logcommand.LogCommand): class Verify(logcommand.LogCommand): + usage = '[CUEFILE]...' summary = "verify image" + description = ''' +Verifies the image from the given .cue files against the AccurateRip database. +''' + def do(self, args): prog = program.Program() runner = task.SyncRunner() From d7848cd34c5eb5968a25b51cee8509ab6f15fa0a Mon Sep 17 00:00:00 2001 From: Velo Superman Date: Fri, 1 Mar 2013 19:32:15 +0100 Subject: [PATCH 010/132] break into two lines to separate logger problems from morituri --- morituri/common/program.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index 5c49d86..5baf0e6 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -698,7 +698,8 @@ class Program(log.Loggable): def writeLog(self, discName, logger): logPath = '%s.log' % discName handle = open(logPath, 'w') - handle.write(logger.log(self.result).encode('utf-8')) + log = logger.log(self.result) + handle.write(log.encode('utf-8')) handle.close() self.logPath = logPath From 18cb676d81df0ca2130feac1c71a94a47c2959e1 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 2 Mar 2013 11:57:52 +0100 Subject: [PATCH 011/132] Use all but last track to find offset. Fixes problems for people whose reader doesn't do overread on the last track. --- morituri/rip/offset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/morituri/rip/offset.py b/morituri/rip/offset.py index 43e3131..bc7ca1d 100644 --- a/morituri/rip/offset.py +++ b/morituri/rip/offset.py @@ -170,8 +170,9 @@ CD in the AccurateRip database.""" 'Offset of device is likely %d, confirming ...\n' % offset) - # now try and rip all other tracks as well - for track in range(2, len(table.tracks) + 1): + # now try and rip all other tracks as well, except for the + # last one (to avoid readers that can't do overread + for track in range(2, (len(table.tracks) + 1) - 1): try: archecksum = self._arcs(runner, table, track, offset) except task.TaskException, e: From a2afe796441663c1b94dfdbd39e568a439ee5262 Mon Sep 17 00:00:00 2001 From: Velo Superman Date: Sun, 3 Mar 2013 14:19:23 +0100 Subject: [PATCH 012/132] handle encoding properly when outputting diff --- morituri/test/common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/morituri/test/common.py b/morituri/test/common.py index d1aa8d7..97e716e 100644 --- a/morituri/test/common.py +++ b/morituri/test/common.py @@ -26,11 +26,13 @@ def _diff(old, new, desc): raise AssertionError( ("\nError while comparing strings:\n" - "%s") % (output, )) + "%s") % (output.encode('utf-8'), )) def diffStrings(orig, new, desc='input'): + assert type(orig) == type(new) + def _tolines(s): return [line + '\n' for line in s.split('\n')] From 7e3d7f0ab77bb6d8d3a967f4ce31c6b4ba23c89f Mon Sep 17 00:00:00 2001 From: Velo Superman Date: Sun, 3 Mar 2013 17:55:10 +0100 Subject: [PATCH 013/132] return cdrdao version used --- morituri/common/program.py | 5 +++-- morituri/rip/cd.py | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index 5baf0e6..bac1d69 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -101,8 +101,9 @@ class Program(log.Loggable): def getFastToc(self, runner, toc_pickle, device): """ Retrieve the normal TOC table from a toc pickle or the drive. + Also retrieves the cdrdao version - @rtype: L{table.Table} + @rtype: tuple of L{table.Table}, str """ def function(r, t): r.run(t) @@ -132,7 +133,7 @@ class Program(log.Loggable): ptoc.persist(t.table) toc = ptoc.object assert toc.hasTOC() - return toc + return (toc, version) def getTable(self, runner, cddbdiscid, mbdiscid, device): """ diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 436b6ba..e6e888a 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -64,10 +64,8 @@ class _CD(logcommand.LogCommand): self.program.loadDevice(self.device) self.program.unmountDevice(self.device) - version = None - # first, read the normal TOC, which is fast - self.ittoc = self.program.getFastToc(self.runner, + self.ittoc, version = self.program.getFastToc(self.runner, self.options.toc_pickle, self.device) From 05ec378c161e7a2bc90aa724ebdcc873eb7d6974 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 3 Mar 2013 18:17:00 +0100 Subject: [PATCH 014/132] add another test case to work on INDEX 02 --- morituri/test/Makefile.am | 1 + morituri/test/surferrosa.toc | 196 ++++++++++++++++++++++++++++++++ morituri/test/test_image_toc.py | 41 +++++++ 3 files changed, 238 insertions(+) create mode 100644 morituri/test/surferrosa.toc diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index 298ada2..8ac84e7 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -48,6 +48,7 @@ EXTRA_DIST = \ cdrdao.readtoc.progress \ silentalarm.result.pickle \ strokes-someday.toc \ + surferrosa.toc \ totbl.fast.toc \ track.flac \ cache/result/fe105a11.pickle \ diff --git a/morituri/test/surferrosa.toc b/morituri/test/surferrosa.toc new file mode 100644 index 0000000..484f83b --- /dev/null +++ b/morituri/test/surferrosa.toc @@ -0,0 +1,196 @@ +CD_DA + +CATALOG "0000000000000" + +// Track 1 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +SILENCE 00:00:32 +FILE "data.wav" 0 03:03:10 +START 00:00:32 + + +// Track 2 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 03:03:10 02:05:00 + + +// Track 3 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 05:08:10 01:48:25 +START 00:00:45 + + +// Track 4 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 06:56:35 01:30:08 + + +// Track 5 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 08:26:43 03:54:70 + + +// Track 6 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 12:21:38 02:31:65 + + +// Track 7 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 14:53:28 03:53:30 + + +// Track 8 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 18:46:58 02:16:55 + + +// Track 9 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 21:03:38 01:52:20 + + +// Track 10 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 22:55:58 01:47:17 + + +// Track 11 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 24:43:00 05:05:63 +INDEX 00:44:70 + + +// Track 12 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 29:48:63 01:42:07 + + +// Track 13 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 31:30:70 02:00:68 + + +// Track 14 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 33:31:63 03:14:25 + + +// Track 15 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 36:46:13 02:53:52 + + +// Track 16 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 39:39:65 01:41:25 + + +// Track 17 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 41:21:15 02:30:00 + + +// Track 18 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 43:51:15 02:14:38 + + +// Track 19 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 46:05:53 02:17:15 + + +// Track 20 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 48:22:68 03:00:57 + + +// Track 21 +TRACK AUDIO +NO COPY +NO PRE_EMPHASIS +TWO_CHANNEL_AUDIO +ISRC "000000000000" +FILE "data.wav" 51:23:50 02:38:38 + diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 0175bed..6485a4c 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -336,3 +336,44 @@ class StrokesTestCase(common.TestCase): t = self.toc.table.tracks[0] self.assertEquals(t.getIndex(0).relative, 0) self.assertEquals(t.getIndex(1).relative, 1) + + +# Surfer Rosa has +# track 00 consisting of 32 frames of SILENCE +# track 11 Vamos with an INDEX 02 + + +class SurferRosaTestCase(common.TestCase): + + def setUp(self): + self.path = os.path.join(os.path.dirname(__file__), + u'surferrosa.toc') + self.toc = toc.TocFile(self.path) + self.toc.parse() + self.assertEquals(len(self.toc.table.tracks), 21) + + def testIndexes(self): + # HTOA + t = self.toc.table.tracks[0] + self.assertEquals(len(t.indexes), 2) + self.assertEquals(t.getIndex(0).relative, 0) + self.assertEquals(t.getIndex(0).absolute, 0) + self.assertEquals(t.getIndex(1).relative, 32) + self.assertEquals(t.getIndex(1).absolute, 32) + + # track 11, Vamos + + t = self.toc.table.tracks[10] + self.assertEquals(len(t.indexes), 2) + + # 32 frames of silence, and 1483 seconds of data.wav + self.assertEquals(t.getIndex(1).relative, 111257) + self.assertEquals(t.getIndex(1).absolute, 111257) + self.assertEquals(t.getIndex(2).relative, 3370) + self.assertEquals(t.getIndex(2).absolute, None) + + self.toc.table.absolutize() + self.assertEquals(t.getIndex(2).absolute, 3370) + +# print self.toc.table.cue() + From 70d39feee9db6a85bfe3b4315ff2813f8018cf40 Mon Sep 17 00:00:00 2001 From: dioltas Date: Sun, 3 Mar 2013 23:17:20 +0000 Subject: [PATCH 015/132] Compare AccurateRip to num tracks -1, as last track not being checked --- morituri/rip/offset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/rip/offset.py b/morituri/rip/offset.py index bc7ca1d..26c14b2 100644 --- a/morituri/rip/offset.py +++ b/morituri/rip/offset.py @@ -188,7 +188,7 @@ CD in the AccurateRip database.""" track, i)) count += 1 - if count == len(table.tracks): + if count == len(table.tracks) - 1: self._foundOffset(device, offset) return 0 else: From ee99d833a7e8d11d634d9ea1f9193c8be340a88c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 17 Feb 2013 18:30:32 +0100 Subject: [PATCH 016/132] rename our musicbrainzngs.py to mbngs.py Helps Debian package which prefers to strip out our copy of musicbrainzngs --- morituri/common/Makefile.am | 2 +- morituri/common/{musicbrainzngs.py => mbngs.py} | 10 +++++----- morituri/common/program.py | 8 ++++---- morituri/rip/debug.py | 4 ++-- morituri/test/Makefile.am | 2 +- ...t_common_musicbrainzngs.py => test_common_mbngs.py} | 6 +++--- morituri/test/test_common_program.py | 6 +++--- 7 files changed, 19 insertions(+), 19 deletions(-) rename morituri/common/{musicbrainzngs.py => mbngs.py} (95%) rename morituri/test/{test_common_musicbrainzngs.py => test_common_mbngs.py} (69%) diff --git a/morituri/common/Makefile.am b/morituri/common/Makefile.am index ce1e3a0..bbda7f6 100644 --- a/morituri/common/Makefile.am +++ b/morituri/common/Makefile.am @@ -16,7 +16,7 @@ morituri_PYTHON = \ gstreamer.py \ log.py \ logcommand.py \ - musicbrainzngs.py \ + mbngs.py \ program.py \ renamer.py \ task.py diff --git a/morituri/common/musicbrainzngs.py b/morituri/common/mbngs.py similarity index 95% rename from morituri/common/musicbrainzngs.py rename to morituri/common/mbngs.py index 9a0ecfa..055899c 100644 --- a/morituri/common/musicbrainzngs.py +++ b/morituri/common/mbngs.py @@ -1,4 +1,4 @@ -# -*- Mode: Python; test-case-name: morituri.test.test_common_musicbrainzngs -*- +# -*- Mode: Python; test-case-name: morituri.test.test_common_mbngs -*- # vi:si:et:sw=4:sts=4:ts=4 # Morituri - for those about to RIP @@ -90,7 +90,7 @@ def _record(record, which, name, what): handle = open(filename, 'w') handle.write(json.dumps(what)) handle.close() - log.info('musicbrainzngs', 'Wrote %s %s to %s', which, name, filename) + log.info('mbngs', 'Wrote %s %s to %s', which, name, filename) def _getMetadata(releaseShort, release, discid): @@ -117,7 +117,7 @@ def _getMetadata(releaseShort, release, discid): artist = credit[0]['artist'] if len(credit) > 1: - log.debug('musicbrainzngs', 'artist-credit more than 1: %r', credit) + log.debug('mbngs', 'artist-credit more than 1: %r', credit) for i, c in enumerate(credit): if isinstance(c, dict): @@ -136,7 +136,7 @@ def _getMetadata(releaseShort, release, discid): metadata.sortName = artist['sort-name'] # FIXME: is format str ? if not 'date' in release: - log.warning('musicbrainzngs', 'Release %r does not have date', release) + log.warning('mbngs', 'Release %r does not have date', release) else: metadata.release = release['date'] @@ -170,7 +170,7 @@ def _getMetadata(releaseShort, release, discid): track = TrackMetadata() credit = t['recording']['artist-credit'] if len(credit) > 1: - log.debug('musicbrainzngs', + log.debug('mbngs', 'artist-credit more than 1: %r', credit) # credit is of the form [dict, str, dict, ... ] for i, c in enumerate(credit): diff --git a/morituri/common/program.py b/morituri/common/program.py index 5c49d86..73385b9 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -28,7 +28,7 @@ import os import sys import time -from morituri.common import common, log, musicbrainzngs, cache +from morituri.common import common, log, mbngs, cache from morituri.program import cdrdao, cdparanoia from morituri.image import image @@ -301,11 +301,11 @@ class Program(log.Loggable): for _ in range(0, 4): try: - metadatas = musicbrainzngs.musicbrainz(mbdiscid, + metadatas = mbngs.musicbrainz(mbdiscid, record=self._record) - except musicbrainzngs.NotFoundException, e: + except mbngs.NotFoundException, e: break - except musicbrainzngs.MusicBrainzException, e: + except mbngs.MusicBrainzException, e: self._stdout.write("Warning: %r\n" % (e, )) time.sleep(5) continue diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index e7f09d0..d8d27d6 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -196,8 +196,8 @@ Example disc id: KnpGsLhvH.lPrNc1PBL21lb9Bg4-""" self.stdout.write('Please specify a MusicBrainz disc id.\n') return 3 - from morituri.common import musicbrainzngs - metadatas = musicbrainzngs.musicbrainz(discId) + from morituri.common import mbngs + metadatas = mbngs.musicbrainz(discId) self.stdout.write('%d releases\n' % len(metadatas)) for i, md in enumerate(metadatas): diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index 8ac84e7..b32aed1 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -11,7 +11,7 @@ EXTRA_DIST = \ test_common_drive.py \ test_common_encode.py \ test_common_gstreamer.py \ - test_common_musicbrainzngs.py \ + test_common_mbngs.py \ test_common_program.py \ test_common_renamer.py \ test_image_cue.py \ diff --git a/morituri/test/test_common_musicbrainzngs.py b/morituri/test/test_common_mbngs.py similarity index 69% rename from morituri/test/test_common_musicbrainzngs.py rename to morituri/test/test_common_mbngs.py index ffe9c1b..ba5f30b 100644 --- a/morituri/test/test_common_musicbrainzngs.py +++ b/morituri/test/test_common_mbngs.py @@ -1,4 +1,4 @@ -# -*- Mode: Python; test-case-name: morituri.test.test_common_musicbrainzngs -*- +# -*- Mode: Python; test-case-name: morituri.test.test_common_mbngs -*- # vi:si:et:sw=4:sts=4:ts=4 import os @@ -6,7 +6,7 @@ import json import unittest -from morituri.common import musicbrainzngs +from morituri.common import mbngs class MetadataTestCase(unittest.TestCase): @@ -19,6 +19,6 @@ class MetadataTestCase(unittest.TestCase): handle.close() discid = "wbjbST2jUHRZaB1inCyxxsL7Eqc-" - metadata = musicbrainzngs._getMetadata({}, response['release'], discid) + metadata = mbngs._getMetadata({}, response['release'], discid) self.failIf(metadata.release) diff --git a/morituri/test/test_common_program.py b/morituri/test/test_common_program.py index 102540b..21a8926 100644 --- a/morituri/test/test_common_program.py +++ b/morituri/test/test_common_program.py @@ -7,7 +7,7 @@ import pickle import unittest from morituri.result import result -from morituri.common import program, accurip, musicbrainzngs +from morituri.common import program, accurip, mbngs from morituri.rip import common as rcommon @@ -95,7 +95,7 @@ class PathTestCase(unittest.TestCase): def testStandardTemplateFilled(self): prog = program.Program() - md = musicbrainzngs.DiscMetadata() + md = mbngs.DiscMetadata() md.artist = md.sortName = 'Jeff Buckley' md.title = 'Grace' prog.metadata = md @@ -107,7 +107,7 @@ class PathTestCase(unittest.TestCase): def testIssue66TemplateFilled(self): prog = program.Program() - md = musicbrainzngs.DiscMetadata() + md = mbngs.DiscMetadata() md.artist = md.sortName = 'Jeff Buckley' md.title = 'Grace' prog.metadata = md From 307db4f59f8639d6837665e0a6d4fe6f47970256 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 5 Mar 2013 16:59:38 +0100 Subject: [PATCH 017/132] update logging; bump classVersion of Table now table will get a logName set correctly when unpickled too --- morituri/common/program.py | 6 +++++- morituri/image/table.py | 8 ++++++-- morituri/image/toc.py | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index 73385b9..af17c92 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -150,8 +150,12 @@ class Program(log.Loggable): t = cdrdao.ReadTableTask(device=device) runner.run(t) ptable.persist(t.table) + self.debug('getTable: read table %r' % t.table) else: - self.debug('getTable: cddbdiscid %s in cache' % cddbdiscid) + self.debug('getTable: cddbdiscid %s, mbdiscid %s in cache' % ( + cddbdiscid, mbdiscid)) + ptable.object.unpickled() + self.debug('getTable: loaded table %r' % ptable.object) itable = ptable.object assert itable.hasTOC() diff --git a/morituri/image/table.py b/morituri/image/table.py index 18e9efb..9ea055c 100644 --- a/morituri/image/table.py +++ b/morituri/image/table.py @@ -168,7 +168,7 @@ class Table(object, log.Loggable): catalog = None # catalog number; FIXME: is this UPC ? cdtext = None - classVersion = 2 + classVersion = 3 def __init__(self, tracks=None): if not tracks: @@ -176,10 +176,14 @@ class Table(object, log.Loggable): self.tracks = tracks self.cdtext = {} - self.logName = "Table 0x%08X" % id(self) # done this way because just having a class-defined instance var # gets overridden when unpickling self.instanceVersion = self.classVersion + self.unpickled() + + def unpickled(self): + self.logName = "Table 0x%08x v%d" % (id(self), self.instanceVersion) + self.debug('set logName') def getTrackStart(self, number): """ diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 9382dc2..f84227c 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -305,7 +305,7 @@ class TocFile(object, log.Loggable): # totalLength was added up to the penultimate track self.table.leadout = totalLength + currentLength - self.debug('leadout: %r', self.table.leadout) + self.debug('parse: leadout: %r', self.table.leadout) def message(self, number, message): """ From 330b2dcb6cbc567f428154b91f82ea0a10c61cb9 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 5 Mar 2013 17:07:36 +0100 Subject: [PATCH 018/132] make track/index debug uniform --- morituri/image/toc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index f84227c..b43ee41 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -170,7 +170,7 @@ class TocFile(object, log.Loggable): relative=relativeOffset + pregapLength, counter=counter) self.debug( - '[track %2d index 01] pregapLength %r, added %r', + '[track %02d index 01] pregapLength %r, added %r', currentTrack.number, pregapLength, currentTrack.getIndex(1)) @@ -291,7 +291,7 @@ class TocFile(object, log.Loggable): offset = common.msfToFrames(m.group('offset')) currentTrack.index(indexNumber, path=currentFile.path, relative=offset, counter=counter) - self.debug('[track %2d index %2d] added %r', + self.debug('[track %02d index %02d] added %r', currentTrack.number, indexNumber, currentTrack.getIndex(indexNumber)) @@ -300,7 +300,7 @@ class TocFile(object, log.Loggable): currentTrack.index(1, path=currentFile.path, absolute=absoluteOffset + pregapLength, relative=relativeOffset + pregapLength, counter=counter) - self.debug('[track %2d index 01] last track, added %r', + self.debug('[track %02d index 01] last track, added %r', currentTrack.number, currentTrack.getIndex(1)) # totalLength was added up to the penultimate track From 20f6a0b70ee9012ab43469151a6c942de323f7ca Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 5 Mar 2013 17:15:54 +0100 Subject: [PATCH 019/132] improve debug --- morituri/program/cdrdao.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/morituri/program/cdrdao.py b/morituri/program/cdrdao.py index 15f90bf..669f0d0 100644 --- a/morituri/program/cdrdao.py +++ b/morituri/program/cdrdao.py @@ -290,7 +290,7 @@ class DiscInfoTask(CDRDAOTask): @param device: the device to rip from @type device: str """ - self.debug('creating DiscInfoTask') + self.debug('creating DiscInfoTask for device %r', device) CDRDAOTask.__init__(self) self.options = ['disk-info', ] @@ -342,6 +342,8 @@ class ReadSessionTask(CDRDAOTask): @param device: the device to rip from @type device: str """ + self.debug('Creating ReadSessionTask for session %d on device %r', + session, device) CDRDAOTask.__init__(self) self.parser = OutputParser(self) (fd, self._tocfilepath) = tempfile.mkstemp( From 35f568181ffe653d0619ca3e43c12b5662df1be6 Mon Sep 17 00:00:00 2001 From: dioltas Date: Wed, 6 Mar 2013 19:12:03 +0000 Subject: [PATCH 020/132] Convert values returned from pycdio to str (workaround for upstream bug) --- morituri/common/drive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/morituri/common/drive.py b/morituri/common/drive.py index 80c3de6..a206ac0 100644 --- a/morituri/common/drive.py +++ b/morituri/common/drive.py @@ -34,7 +34,8 @@ def _listify(listOrString): def getAllDevicePaths(): try: - return _getAllDevicePathsPyCdio() + # see https://savannah.gnu.org/bugs/index.php?38477 + return [str(dev) for dev in _getAllDevicePathsPyCdio()] except ImportError: log.info('drive', 'Cannot import pycdio') return _getAllDevicePathsStatic() From 47a9481bc1b9825a659ec2ed747070b74b21b1e6 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Fri, 10 May 2013 22:06:04 +0200 Subject: [PATCH 021/132] Document and fix comments. --- morituri/common/checksum.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/morituri/common/checksum.py b/morituri/common/checksum.py index dea979f..d424175 100644 --- a/morituri/common/checksum.py +++ b/morituri/common/checksum.py @@ -209,6 +209,13 @@ class ChecksumTask(log.Loggable, gstreamer.GstPipelineTask): def do_checksum_buffer(self, buf, checksum): """ Subclasses should implement this. + + @param buf: a byte buffer containing two 16-bit samples per + channel. + @type buf: C{str} + @param checksum: the checksum so far, as returned by the + previous call. + @type checksum: C{int} """ raise NotImplementedError @@ -240,7 +247,7 @@ class ChecksumTask(log.Loggable, gstreamer.GstPipelineTask): sample = self._first + self._bytes / 4 samplesDone = sample - self._sampleStart progress = float(samplesDone) / float((self._sampleLength)) - # marshall to the main thread + # marshal to the main thread self.schedule(0, self.setProgress, progress) def _eos_cb(self, sink): From 66abdfb21455f4fc7d61eb78bccd108d70314825 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Fri, 10 May 2013 22:06:29 +0200 Subject: [PATCH 022/132] Accept more than one file for checksum. --- morituri/rip/debug.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index d8d27d6..96b14e6 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -100,21 +100,22 @@ class Checksum(logcommand.LogCommand): summary = "run a checksum task" def do(self, args): - try: - fromPath = unicode(args[0]) - except IndexError: - self.stdout.write('Please specify an input file.\n') + if not args: + self.stdout.write('Please specify one or more input files.\n') return 3 runner = task.SyncRunner() - # here to avoid import gst eating our options from morituri.common import checksum - checksumtask = checksum.CRC32Task(fromPath) - runner.run(checksumtask) + for arg in args: + fromPath = unicode(args[0]) - self.stdout.write('Checksum: %08x\n' % checksumtask.checksum) + checksumtask = checksum.CRC32Task(fromPath) + + runner.run(checksumtask) + + self.stdout.write('Checksum: %08x\n' % checksumtask.checksum) class Encode(logcommand.LogCommand): From 32000bd0d0d4f2626cd5b742c1c7ea525a39fa83 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Fri, 10 May 2013 22:06:38 +0200 Subject: [PATCH 023/132] Add new command: rip debug maxsample Calculates the maximum absolute sample value. Helpful to verify whether HTOA tracks are pure silence or not. --- morituri/common/checksum.py | 17 +++++++++++++++++ morituri/rip/debug.py | 28 +++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/morituri/common/checksum.py b/morituri/common/checksum.py index d424175..330af9d 100644 --- a/morituri/common/checksum.py +++ b/morituri/common/checksum.py @@ -387,3 +387,20 @@ class TRMTask(task.GstPipelineTask): def stopped(self): self.trm = self._trm + +class MaxSampleTask(ChecksumTask): + """ + I check for the biggest sample value. + """ + + description = 'Finding highest sample value' + + def do_checksum_buffer(self, buf, checksum): + values = struct.unpack("<%dh" % (len(buf) / 2), buf) + absvalues = [abs(v) for v in values] + m = max(absvalues) + if checksum < m: + checksum = m + + return checksum + diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index 96b14e6..9751aeb 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -159,6 +159,32 @@ class Encode(logcommand.LogCommand): self.stdout.write('Peak level: %r\n' % encodetask.peak) self.stdout.write('Encoded to %s\n' % toPath.encode('utf-8')) + +class MaxSample(logcommand.LogCommand): + + summary = "run a max sample task" + + def do(self, args): + if not args: + self.stdout.write('Please specify one or more input files.\n') + return 3 + + runner = task.SyncRunner() + # here to avoid import gst eating our options + from morituri.common import checksum + + for arg in args: + fromPath = unicode(arg.decode('utf-8')) + + checksumtask = checksum.MaxSampleTask(fromPath) + + runner.run(checksumtask) + + self.stdout.write('%s\n' % arg) + self.stdout.write('Biggest absolute sample: %04x\n' % + checksumtask.checksum) + + class Tag(logcommand.LogCommand): summary = "run a tag reading task" @@ -223,4 +249,4 @@ class Debug(logcommand.LogCommand): summary = "debug internals" - subCommandClasses = [Checksum, Encode, Tag, MusicBrainzNGS, ResultCache] + subCommandClasses = [Checksum, Encode, MaxSample, Tag, MusicBrainzNGS, ResultCache] From d9e2175ddce1588bdf20c584a2aa0d209291808b Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Wed, 3 Jul 2013 20:15:32 +0200 Subject: [PATCH 024/132] Handle unknown option on rip cd info. Fixes #30. --- morituri/rip/cd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 436b6ba..e067b67 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -90,7 +90,8 @@ class _CD(logcommand.LogCommand): if cddbmd: self.stdout.write('FreeDB identifies disc as %s\n' % cddbmd) - if not self.options.unknown: + # also used by rip cd info + if not self.options.get('unknown:', False): self.program.ejectDevice(self.device) return -1 From 27030a9842bbc3b90160a00c1dcabfa0bfbbaadd Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 26 May 2013 21:27:20 +0200 Subject: [PATCH 025/132] fix output --- morituri/common/program.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index af17c92..eecf90d 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -355,7 +355,8 @@ class Program(log.Loggable): metadatas[0].title.encode('utf-8')) elif not metadatas: self._stdout.write( - 'Requested release id %s but none match' % release) + "Requested release id '%s', " + "but none of the found releases match\n" % release) return else: # Select the release that most closely matches the duration. From 1a4f9e367d4e8eb98ed7b1182ee0132898cbf7af Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Thu, 4 Jul 2013 01:02:14 +0200 Subject: [PATCH 026/132] Really handle unknown on rip cd info. Really fixes #30. --- morituri/rip/cd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index e067b67..48a21a4 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -91,7 +91,7 @@ class _CD(logcommand.LogCommand): self.stdout.write('FreeDB identifies disc as %s\n' % cddbmd) # also used by rip cd info - if not self.options.get('unknown:', False): + if not getattr(self.options, 'unknown', False): self.program.ejectDevice(self.device) return -1 From e63fddd77a1e657ca7e0f690f89bedff7fbc4f87 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 11:04:27 +0200 Subject: [PATCH 027/132] cleanup and comment --- morituri/rip/cd.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 48a21a4..2be0af5 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -42,6 +42,10 @@ MAX_TRIES = 5 class _CD(logcommand.LogCommand): + """ + @type program: L{program.Program} + """ + def addOptions(self): # FIXME: have a cache of these pickles somewhere self.parser.add_option('-T', '--toc-pickle', @@ -300,8 +304,11 @@ Log files will log the path to tracks relative to this directory. self.debug('ripIfNotRipped have trackresult, path %r' % trackResult.filename) - path = self.program.getPath(self.program.outdir, self.options.track_template, - self.mbdiscid, number, profile=profile, disambiguate=disambiguate) + '.' + profile.extension + path = self.program.getPath(self.program.outdir, + self.options.track_template, + self.mbdiscid, number, + profile=profile, disambiguate=disambiguate) \ + + '.' + profile.extension self.debug('ripIfNotRipped: path %r' % path) trackResult.number = number From cc2ee6ba077c36c6173fa90786c69d26770c44cb Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 11:04:27 +0200 Subject: [PATCH 028/132] cleanup and comment --- morituri/rip/cd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 2be0af5..3ccaf8b 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -28,7 +28,7 @@ import gobject gobject.threads_init() from morituri.common import logcommand, common, accurip, gstreamer -from morituri.common import drive, program, cache +from morituri.common import drive, program from morituri.result import result from morituri.program import cdrdao, cdparanoia from morituri.rip import common as rcommon From 7813103a4dacf35f10cb3f62a8f6908c1e3689eb Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 11:31:27 +0200 Subject: [PATCH 029/132] get version for cdrdao too. Add debug commands: rip debug version cdrdao/cdparanoia --- morituri/common/common.py | 44 +++++++++++++++++++++++- morituri/program/cdparanoia.py | 23 +++---------- morituri/program/cdrdao.py | 13 +++++++ morituri/rip/cd.py | 5 +-- morituri/rip/debug.py | 26 +++++++++++++- morituri/test/test_program_cdparanoia.py | 6 +++- morituri/test/test_program_cdrdao.py | 14 ++++++-- 7 files changed, 106 insertions(+), 25 deletions(-) diff --git a/morituri/common/common.py b/morituri/common/common.py index 0d91ed2..6608b7d 100644 --- a/morituri/common/common.py +++ b/morituri/common/common.py @@ -24,8 +24,9 @@ import os import os.path import math +import subprocess - +from morituri.extern import asyncsub from morituri.extern.log import log FRAMES_PER_SECOND = 75 @@ -288,3 +289,44 @@ def getRelativePath(targetPath, collectionPath): 'getRelativePath: target and collection in different dir, %r' % rel) return os.path.join(rel, os.path.basename(targetPath)) + + +class VersionGetter(object): + """ + I get the version of a program by looking for it in command output + according to a regexp. + """ + + def __init__(self, dependency, args, regexp, expander): + """ + @param dependency: name of the dependency providing the program + @param args: the arguments to invoke to show the version + @type args: list of str + @param regexp: the regular expression to get the version + @param expander: the expansion string for the version using the + regexp group dict + """ + + self._dep = dependency + self._args = args + self._regexp = regexp + self._expander = expander + + def get(self): + version = "(Unknown)" + + try: + p = asyncsub.Popen(self._args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True) + output = asyncsub.recv_some(p, e=0, stderr=1) + vre = self._regexp.search(output) + if vre: + version = self._expander % vre.groupdict() + except OSError, e: + import errno + if e.errno == errno.ENOENT: + raise MissingDependencyException(self._dep) + raise + + return version diff --git a/morituri/program/cdparanoia.py b/morituri/program/cdparanoia.py index e8f6c59..46176d5 100644 --- a/morituri/program/cdparanoia.py +++ b/morituri/program/cdparanoia.py @@ -550,25 +550,12 @@ _VERSION_RE = re.compile( def getCdParanoiaVersion(): - version = "(Unknown)" + getter = common.VersionGetter('cdparanoia', + ["cdparanoia", "-V"], + _VERSION_RE, + "%(version)s %(release)s") - try: - p = asyncsub.Popen(["cdparanoia", "-V"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, close_fds=True) - version = asyncsub.recv_some(p, e=0, stderr=1) - vre = _VERSION_RE.search(version) - if vre and len(vre.groups()) == 2: - version = "%s %s" % ( - vre.groupdict().get('version'), - vre.groupdict().get('release')) - except OSError, e: - import errno - if e.errno == errno.ENOENT: - raise common.MissingDependencyException('cdparanoia') - raise - - return version + return getter.get() _OK_RE = re.compile(r'Drive tests OK with Paranoia.') diff --git a/morituri/program/cdrdao.py b/morituri/program/cdrdao.py index 669f0d0..c6fba64 100644 --- a/morituri/program/cdrdao.py +++ b/morituri/program/cdrdao.py @@ -507,3 +507,16 @@ class ProgramFailedException(Exception): def __init__(self, code): self.code = code self.args = (code, ) + + +_VERSION_RE = re.compile( + "^Cdrdao version (?P.+) -") + + +def getCDRDAOVersion(): + getter = common.VersionGetter('cdrdao', + ["cdrdao"], + _VERSION_RE, + "%(version)s") + + return getter.get() diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 3ccaf8b..73124d8 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -117,8 +117,9 @@ class _CD(logcommand.LogCommand): # result - self.program.result.cdrdaoVersion = version - self.program.result.cdparanoiaVersion = cdparanoia.getCdParanoiaVersion() + self.program.result.cdrdaoVersion = cdrdao.getCDRDAOVersion() + self.program.result.cdparanoiaVersion = \ + cdparanoia.getCdParanoiaVersion() info = drive.getDeviceInfo(self.parentCommand.options.device) if info: try: diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index 9751aeb..e182109 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -245,8 +245,32 @@ Example disc id: KnpGsLhvH.lPrNc1PBL21lb9Bg4-""" track.title.encode('utf-8'))) +class CDParanoia(logcommand.LogCommand): + + def do(self, args): + from morituri.program import cdparanoia + version = cdparanoia.getCdParanoiaVersion() + self.stdout.write("cdparanoia version: %s\n" % version) + + +class CDRDAO(logcommand.LogCommand): + + def do(self, args): + from morituri.program import cdrdao + version = cdrdao.getCDRDAOVersion() + self.stdout.write("cdrdao version: %s\n" % version) + + +class Version(logcommand.LogCommand): + + summary = "debug version getting" + + subCommandClasses = [CDParanoia, CDRDAO] + + class Debug(logcommand.LogCommand): summary = "debug internals" - subCommandClasses = [Checksum, Encode, MaxSample, Tag, MusicBrainzNGS, ResultCache] + subCommandClasses = [Checksum, Encode, MaxSample, Tag, MusicBrainzNGS, + ResultCache, Version] diff --git a/morituri/test/test_program_cdparanoia.py b/morituri/test/test_program_cdparanoia.py index 397073d..4d7b510 100644 --- a/morituri/test/test_program_cdparanoia.py +++ b/morituri/test/test_program_cdparanoia.py @@ -65,7 +65,11 @@ class ErrorTestCase(common.TestCase): class VersionTestCase(common.TestCase): def testGetVersion(self): - self.failUnless(cdparanoia.getCdParanoiaVersion()) + v = cdparanoia.getCdParanoiaVersion() + self.failUnless(v) + # of the form III 10.2 + # make sure it ends with a digit + self.failUnless(int(v[-1])) class AnalyzeFileTask(cdparanoia.AnalyzeTask): diff --git a/morituri/test/test_program_cdrdao.py b/morituri/test/test_program_cdrdao.py index 0a777dd..37b4986 100644 --- a/morituri/test/test_program_cdrdao.py +++ b/morituri/test/test_program_cdrdao.py @@ -2,10 +2,11 @@ # vi:si:et:sw=4:sts=4:ts=4 import os -import unittest from morituri.program import cdrdao +from morituri.test import common + class FakeTask: @@ -13,7 +14,7 @@ class FakeTask: pass -class ParseTestCase(unittest.TestCase): +class ParseTestCase(common.TestCase): def setUp(self): path = os.path.join(os.path.dirname(__file__), @@ -34,3 +35,12 @@ class ParseTestCase(unittest.TestCase): self.assertEquals(track.getIndex(1).absolute, offset) self.assertEquals(self._parser.version, '1.2.2') + + +class VersionTestCase(common.TestCase): + + def testGetVersion(self): + v = cdrdao.getCDRDAOVersion() + self.failUnless(v) + # make sure it starts with a digit + self.failUnless(int(v[0])) From 54f7bedae4185f797d05dbdc94d301b41adfc7f7 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 11:41:45 +0200 Subject: [PATCH 030/132] 80 character limit --- morituri/rip/cd.py | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 73124d8..cf03b46 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -84,7 +84,8 @@ class _CD(logcommand.LogCommand): self.stdout.write("MusicBrainz lookup URL %s\n" % self.ittoc.getMusicBrainzSubmitURL()) - self.program.metadata = self.program.getMusicBrainz(self.ittoc, self.mbdiscid, + self.program.metadata = self.program.getMusicBrainz(self.ittoc, + self.mbdiscid, release=self.options.release_id) if not self.program.metadata: @@ -108,10 +109,13 @@ class _CD(logcommand.LogCommand): assert self.itable.getCDDBDiscId() == self.ittoc.getCDDBDiscId(), \ "full table's id %s differs from toc id %s" % ( self.itable.getCDDBDiscId(), self.ittoc.getCDDBDiscId()) - assert self.itable.getMusicBrainzDiscId() == self.ittoc.getMusicBrainzDiscId(), \ + assert self.itable.getMusicBrainzDiscId() == \ + self.ittoc.getMusicBrainzDiscId(), \ "full table's mb id %s differs from toc id mb %s" % ( - self.itable.getMusicBrainzDiscId(), self.ittoc.getMusicBrainzDiscId()) - assert self.itable.getAccurateRipURL() == self.ittoc.getAccurateRipURL(), \ + self.itable.getMusicBrainzDiscId(), + self.ittoc.getMusicBrainzDiscId()) + assert self.itable.getAccurateRipURL() == \ + self.ittoc.getAccurateRipURL(), \ "full table's AR URL %s differs from toc AR URL %s" % ( self.itable.getAccurateRipURL(), self.ittoc.getAccurateRipURL()) @@ -123,18 +127,21 @@ class _CD(logcommand.LogCommand): info = drive.getDeviceInfo(self.parentCommand.options.device) if info: try: - self.program.result.cdparanoiaDefeatsCache = self.getRootCommand( - ).config.getDefeatsCache(*info) + self.program.result.cdparanoiaDefeatsCache = \ + self.getRootCommand().config.getDefeatsCache(*info) except KeyError, e: self.debug('Got key error: %r' % (e, )) - self.program.result.artist = self.program.metadata and self.program.metadata.artist \ + self.program.result.artist = self.program.metadata \ + and self.program.metadata.artist \ or 'Unknown Artist' - self.program.result.title = self.program.metadata and self.program.metadata.title \ + self.program.result.title = self.program.metadata \ + and self.program.metadata.title \ or 'Unknown Title' # cdio is optional for now try: import cdio - _, self.program.result.vendor, self.program.result.model, self.program.result.release = \ + _, self.program.result.vendor, self.program.result.model, \ + self.program.result.release = \ cdio.Device(self.device).get_hwinfo() except ImportError: self.stdout.write( @@ -266,8 +273,9 @@ Log files will log the path to tracks relative to this directory. ### write disc files disambiguate = False while True: - discName = self.program.getPath(self.program.outdir, self.options.disc_template, - self.mbdiscid, 0, profile=profile, disambiguate=disambiguate) + discName = self.program.getPath(self.program.outdir, + self.options.disc_template, self.mbdiscid, 0, + profile=profile, disambiguate=disambiguate) dirname = os.path.dirname(discName) if os.path.exists(dirname): self.stdout.write("Output directory %s already exists\n" % @@ -415,8 +423,9 @@ Log files will log the path to tracks relative to this directory. ripIfNotRipped(i + 1) ### write disc files - discName = self.program.getPath(self.program.outdir, self.options.disc_template, - self.mbdiscid, 0, profile=profile, disambiguate=disambiguate) + discName = self.program.getPath(self.program.outdir, + self.options.disc_template, self.mbdiscid, 0, + profile=profile, disambiguate=disambiguate) dirname = os.path.dirname(discName) if not os.path.exists(dirname): os.makedirs(dirname) @@ -446,8 +455,10 @@ Log files will log the path to tracks relative to this directory. if not track.audio: continue - path = self.program.getPath(self.program.outdir, self.options.track_template, - self.mbdiscid, i + 1, profile=profile, disambiguate=disambiguate) + '.' + profile.extension + path = self.program.getPath(self.program.outdir, + self.options.track_template, self.mbdiscid, i + 1, + profile=profile, + disambiguate=disambiguate) + '.' + profile.extension writeFile(handle, path, self.itable.getTrackLength(i + 1) / common.FRAMES_PER_SECOND) @@ -475,7 +486,8 @@ Log files will log the path to tracks relative to this directory. self.program.verifyImage(self.runner, responses) - self.stdout.write("\n".join(self.program.getAccurateRipResults()) + "\n") + self.stdout.write("\n".join( + self.program.getAccurateRipResults()) + "\n") self.program.saveRipResult() From 7ba5f24f7ac7cd964c7bf641f1c6ca407de0eec8 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 17:32:59 +0200 Subject: [PATCH 031/132] we already get the version some other way --- morituri/common/program.py | 2 +- morituri/rip/cd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index 1b7ea61..685a3ad 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -133,7 +133,7 @@ class Program(log.Loggable): ptoc.persist(t.table) toc = ptoc.object assert toc.hasTOC() - return (toc, version) + return toc def getTable(self, runner, cddbdiscid, mbdiscid, device): """ diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 444ee18..c2acb28 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -69,7 +69,7 @@ class _CD(logcommand.LogCommand): self.program.unmountDevice(self.device) # first, read the normal TOC, which is fast - self.ittoc, version = self.program.getFastToc(self.runner, + self.ittoc = self.program.getFastToc(self.runner, self.options.toc_pickle, self.device) From f93390d1bc1545513caa3a6c77e3a246bb7f0324 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 18:37:10 +0200 Subject: [PATCH 032/132] pass config to program --- morituri/common/program.py | 4 +++- morituri/rip/cd.py | 3 ++- morituri/rip/image.py | 8 ++++---- morituri/test/test_common_program.py | 13 +++++++------ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index 685a3ad..08a1843 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -49,6 +49,7 @@ class Program(log.Loggable): @ivar result: the rip's result @type result: L{result.RipResult} @type outdir: unicode + @type config: L{morituri.common.config.Config} """ cuePath = None @@ -59,13 +60,14 @@ class Program(log.Loggable): _stdout = None - def __init__(self, record=False, stdout=sys.stdout): + def __init__(self, config, record=False, stdout=sys.stdout): """ @param record: whether to record results of API calls for playback. """ self._record = record self._cache = cache.ResultCache() self._stdout = stdout + self._config = config def setWorkingDirectory(self, workingDirectory): if workingDirectory: diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index c2acb28..6b639ef 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -57,7 +57,8 @@ class _CD(logcommand.LogCommand): def do(self, args): - self.program = program.Program(record=self.getRootCommand().record, + self.program = program.Program(self.getRootCommand().config, + record=self.getRootCommand().record, stdout=self.stdout) self.runner = task.SyncRunner() diff --git a/morituri/rip/image.py b/morituri/rip/image.py index 4c19c15..6c766d6 100644 --- a/morituri/rip/image.py +++ b/morituri/rip/image.py @@ -51,7 +51,7 @@ class Encode(logcommand.LogCommand): default=default) def do(self, args): - prog = program.Program() + prog = program.Program(self.getRootCommand().config) prog.outdir = (self.options.output_directory or os.getcwd()) prog.outdir = prog.outdir.decode('utf-8') @@ -110,7 +110,7 @@ class Retag(logcommand.LogCommand): def do(self, args): - prog = program.Program(stdout=self.stdout) + prog = program.Program(self.getRootCommand().config, stdout=self.stdout) runner = task.SyncRunner() for arg in args: @@ -158,7 +158,7 @@ class Rename(logcommand.LogCommand): def do(self, args): - prog = program.Program(stdout=self.stdout) + prog = program.Program(self.getRootCommand.config(), stdout=self.stdout) runner = task.SyncRunner() for arg in args: @@ -208,7 +208,7 @@ Verifies the image from the given .cue files against the AccurateRip database. ''' def do(self, args): - prog = program.Program() + prog = program.Program(self.getRootCommand.config()) runner = task.SyncRunner() cache = accurip.AccuCache() diff --git a/morituri/test/test_common_program.py b/morituri/test/test_common_program.py index 21a8926..d49c8bb 100644 --- a/morituri/test/test_common_program.py +++ b/morituri/test/test_common_program.py @@ -1,13 +1,14 @@ # -*- Mode: Python; test-case-name: morituri.test.test_common_program -*- # vi:si:et:sw=4:sts=4:ts=4 + import os import pickle import unittest from morituri.result import result -from morituri.common import program, accurip, mbngs +from morituri.common import program, accurip, mbngs, config from morituri.rip import common as rcommon @@ -28,7 +29,7 @@ class TrackImageVerifyTestCase(unittest.TestCase): 1842579359, 2850056507, 1329730252, 2526965856, 2525886806, 209743350, 3184062337, 2099956663, 2943874164, 2321637196] - prog = program.Program() + prog = program.Program(config.Config()) prog.result = result.RipResult() # fill it with empty trackresults for i, c in enumerate(checksums): @@ -76,7 +77,7 @@ class HTOATestCase(unittest.TestCase): self._tracks = pickle.load(open(path, 'rb')) def testGetAccurateRipResults(self): - prog = program.Program() + prog = program.Program(config.Config()) prog.result = result.RipResult() prog.result.tracks = self._tracks @@ -86,7 +87,7 @@ class HTOATestCase(unittest.TestCase): class PathTestCase(unittest.TestCase): def testStandardTemplateEmpty(self): - prog = program.Program() + prog = program.Program(config.Config()) path = prog.getPath(u'/tmp', rcommon.DEFAULT_DISC_TEMPLATE, 'mbdiscid', 0) @@ -94,7 +95,7 @@ class PathTestCase(unittest.TestCase): u'/tmp/unknown/Unknown Artist - mbdiscid/Unknown Artist - mbdiscid') def testStandardTemplateFilled(self): - prog = program.Program() + prog = program.Program(config.Config()) md = mbngs.DiscMetadata() md.artist = md.sortName = 'Jeff Buckley' md.title = 'Grace' @@ -106,7 +107,7 @@ class PathTestCase(unittest.TestCase): u'/tmp/unknown/Jeff Buckley - Grace/Jeff Buckley - Grace') def testIssue66TemplateFilled(self): - prog = program.Program() + prog = program.Program(config.Config()) md = mbngs.DiscMetadata() md.artist = md.sortName = 'Jeff Buckley' md.title = 'Grace' From c75f71ead328c193660108eed8176538a1f1e1a3 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 18:38:31 +0200 Subject: [PATCH 033/132] Add path filter --- morituri/common/Makefile.am | 1 + morituri/common/path.py | 44 +++++++++++++++++++++++++++++++ morituri/test/Makefile.am | 1 + morituri/test/test_common_path.py | 16 +++++++++++ 4 files changed, 62 insertions(+) create mode 100644 morituri/common/path.py create mode 100644 morituri/test/test_common_path.py diff --git a/morituri/common/Makefile.am b/morituri/common/Makefile.am index bbda7f6..14c8c95 100644 --- a/morituri/common/Makefile.am +++ b/morituri/common/Makefile.am @@ -17,6 +17,7 @@ morituri_PYTHON = \ log.py \ logcommand.py \ mbngs.py \ + path.py \ program.py \ renamer.py \ task.py diff --git a/morituri/common/path.py b/morituri/common/path.py new file mode 100644 index 0000000..65fc206 --- /dev/null +++ b/morituri/common/path.py @@ -0,0 +1,44 @@ +# -*- Mode: Python; test-case-name: morituri.test.test_common_path -*- +# vi:si:et:sw=4:sts=4:ts=4 + +# Morituri - for those about to RIP + +# Copyright (C) 2009 Thomas Vander Stichele + +# This file is part of morituri. +# +# morituri is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# morituri is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with morituri. If not, see . + +import re + + +class PathFilter(object): + """ + I filter path components for safe storage on file systems. + """ + + def __init__(self, slashes=True, fat=True, special=True): + """ + @param slashes: whether to convert slashes to dashes + @parm fat: whether to strip characters illegal on FAT filesystems + """ + self._slashes = slashes + self._fat = fat + self._special = special + + def filter(self, path): + if self._slashes: + path = re.sub(r'[/]', '-', path, re.UNICODE) + + return path diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index b32aed1..792beb4 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -12,6 +12,7 @@ EXTRA_DIST = \ test_common_encode.py \ test_common_gstreamer.py \ test_common_mbngs.py \ + test_common_path.py \ test_common_program.py \ test_common_renamer.py \ test_image_cue.py \ diff --git a/morituri/test/test_common_path.py b/morituri/test/test_common_path.py new file mode 100644 index 0000000..45b160d --- /dev/null +++ b/morituri/test/test_common_path.py @@ -0,0 +1,16 @@ +# -*- Mode: Python; test-case-name: morituri.test.test_common_path -*- +# vi:si:et:sw=4:sts=4:ts=4 + +from morituri.common import path + +from morituri.test import common + + +class FilterTestCase(common.TestCase): + + def setUp(self): + self._filter = path.PathFilter() + + def testSlash(self): + part = u'A Charm/A Blade' + self.assertEquals(self._filter.filter(part), u'A Charm-A Blade') From e29fa0fb18b8ea4b9de4b8f428016d114872bf7f Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 18:49:00 +0200 Subject: [PATCH 034/132] fix a failing test --- morituri/test/common.py | 3 ++- morituri/test/test_image_toc.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/morituri/test/common.py b/morituri/test/common.py index 97e716e..955db36 100644 --- a/morituri/test/common.py +++ b/morituri/test/common.py @@ -31,7 +31,8 @@ def _diff(old, new, desc): def diffStrings(orig, new, desc='input'): - assert type(orig) == type(new) + assert type(orig) == type(new), 'type %s and %s are different' % ( + type(orig), type(new)) def _tolines(s): return [line + '\n' for line in s.split('\n')] diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 6485a4c..24e9345 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -85,7 +85,8 @@ class CureTestCase(common.TestCase): def testConvertCue(self): self.toc.table.absolutize() cue = self.toc.table.cue() - ref = open(os.path.join(os.path.dirname(__file__), 'cure.cue')).read() + ref = open(os.path.join(os.path.dirname(__file__), 'cure.cue')).read( + ).decode('utf-8') common.diffStrings(cue, ref) # we verify it because it has failed in readdisc in the past From 8e2db21a55e826aa1f43db2856d820c1f298b099 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 18:50:18 +0200 Subject: [PATCH 035/132] Allow getting normal and boolean from config --- morituri/common/config.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/morituri/common/config.py b/morituri/common/config.py index 125f87f..109dfed 100644 --- a/morituri/common/config.py +++ b/morituri/common/config.py @@ -54,6 +54,32 @@ class Config(log.Loggable): self.info('Loaded %d sections from config file' % len(self._parser.sections())) + def write(self): + fd, path = tempfile.mkstemp(suffix=u'.moriturirc') + handle = os.fdopen(fd, 'w') + self._parser.write(handle) + handle.close() + shutil.move(path, self._path) + + + ### any section + + def _getter(self, suffix, section, option): + methodName = 'get' + suffix + method = getattr(self._parser, methodName) + try: + return method(section, option) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return None + + def get(self, section, option): + return self._getter('', section, option) + + def getboolean(self, section, option): + return self._getter('boolean', section, option) + + ### drive sections + def setReadOffset(self, vendor, model, release, offset): """ Set a read offset for the given drive. @@ -96,13 +122,6 @@ class Config(log.Loggable): raise KeyError("Could not find defeats_cache for %s/%s/%s" % ( vendor, model, release)) - def write(self): - fd, path = tempfile.mkstemp(suffix=u'.moriturirc') - handle = os.fdopen(fd, 'w') - self._parser.write(handle) - handle.close() - shutil.move(path, self._path) - def _findDriveSection(self, vendor, model, release): for name in self._parser.sections(): if not name.startswith('drive:'): From 49d8e9c2688b9c5323ec6acf8f76aea2336961c3 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 18:50:52 +0200 Subject: [PATCH 036/132] use PathFilter object to filter paths --- morituri/common/program.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index 08a1843..c5db0b0 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -28,15 +28,11 @@ import os import sys import time -from morituri.common import common, log, mbngs, cache +from morituri.common import common, log, mbngs, cache, path from morituri.program import cdrdao, cdparanoia from morituri.image import image -def filterForPath(text): - return "-".join(text.split("/")) - - # FIXME: should Program have a runner ? @@ -69,6 +65,18 @@ class Program(log.Loggable): self._stdout = stdout self._config = config + d = {} + + for key in ['fat', 'special']: + value = None + value = self._config.getboolean('main', 'path_filter_'+ key) + if value is None: + value = True + + d[key] = value + + self._filter = path.PathFilter(**d) + def setWorkingDirectory(self, workingDirectory): if workingDirectory: self.info('Changing to working directory %s' % workingDirectory) @@ -236,9 +244,9 @@ class Program(log.Loggable): if self.metadata: release = self.metadata.release or '0000' v['y'] = release[:4] - v['A'] = filterForPath(self.metadata.artist) - v['S'] = filterForPath(self.metadata.sortName) - v['d'] = filterForPath(self.metadata.title) + v['A'] = self._filter.filter(self.metadata.artist) + v['S'] = self._filter.filter(self.metadata.sortName) + v['d'] = self._filter.filter(self.metadata.title) v['B'] = self.metadata.barcode v['C'] = self.metadata.catalogNumber if self.metadata.releaseType: @@ -246,16 +254,16 @@ class Program(log.Loggable): v['r'] = self.metadata.releaseType.lower() if i > 0: try: - v['a'] = filterForPath(self.metadata.tracks[i - 1].artist) - v['s'] = filterForPath( + v['a'] = self._filter.filter(self.metadata.tracks[i - 1].artist) + v['s'] = self._filter.filter( self.metadata.tracks[i - 1].sortName) - v['n'] = filterForPath(self.metadata.tracks[i - 1].title) + v['n'] = self._filter.filter(self.metadata.tracks[i - 1].title) except IndexError, e: print 'ERROR: no track %d found, %r' % (i, e) raise else: # htoa defaults to disc's artist - v['a'] = filterForPath(self.metadata.artist) + v['a'] = self._filter.filter(self.metadata.artist) # when disambiguating, use catalogNumber then barcode if disambiguate: From 3ee98930c2c33ed2732c41a88e7b1b540e7a2f6c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 18:55:45 +0200 Subject: [PATCH 037/132] document config file format --- README | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README b/README index e71dd15..191d53d 100644 --- a/README +++ b/README @@ -126,6 +126,28 @@ GOALS - support offline ripping (doing metadata lookup and log rewriting later) - separate the info/result about the rip from the metadata/file generation/... +CONFIGURATION FILE +------------------ + +The configuration file is stored according to +http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html +when possible. + +It lives in $XDG_CONFIG_HOME/morituri/morituri.conf + +The configuration file follows python's ConfigParser syntax. +There is a "main" section and zero or more sections starting with "drive:" + +- main section: + - path_filter_fat: whether to filter path components for FAT file systems + - path_filter_special: whether to filter path components for special + characters + +- drive section: + All these values are probed by morituri and should not be edited by hand. + - defeats_cache: whether this drive can defeat the audio cache + - read_offset: the read offset of the drive + rip command tree ---------------- @@ -155,3 +177,4 @@ rip encode to a different codec retag retag the image with current MusicBrainz data + From 34d58e2b5ccfa4863551aa67fdd1af24a288444d Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 22:46:20 +0200 Subject: [PATCH 038/132] more filtering --- morituri/common/path.py | 20 +++++++++++++++++--- morituri/common/program.py | 7 +++++-- morituri/test/test_common_path.py | 11 ++++++++++- morituri/test/test_program_cdparanoia.py | 2 +- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/morituri/common/path.py b/morituri/common/path.py index 65fc206..b7f39e8 100644 --- a/morituri/common/path.py +++ b/morituri/common/path.py @@ -28,17 +28,31 @@ class PathFilter(object): I filter path components for safe storage on file systems. """ - def __init__(self, slashes=True, fat=True, special=True): + def __init__(self, slashes=True, quotes=True, fat=True, special=False): """ @param slashes: whether to convert slashes to dashes - @parm fat: whether to strip characters illegal on FAT filesystems + @param quotes: whether to normalize quotes + @param fat: whether to strip characters illegal on FAT filesystems + @param special: whether to strip special characters """ self._slashes = slashes + self._quotes = quotes self._fat = fat self._special = special def filter(self, path): if self._slashes: - path = re.sub(r'[/]', '-', path, re.UNICODE) + path = re.sub(r'[/\\]', '-', path, re.UNICODE) + + if self._quotes: + path = re.sub(ur'[\u2019]', "'", path, re.UNICODE) + + if self._special: + # replace separators with a hyphen + path = re.sub(r'[:\|]', '-', path, re.UNICODE) + path = re.sub(r'[\*\?&!\'\"\$\(\)`{}\[\]<>]', '_', path, re.UNICODE) + + if self._fat: + path = re.sub(r'[:\*\?"<>|"]', '_', path, re.UNICODE) return path diff --git a/morituri/common/program.py b/morituri/common/program.py index c5db0b0..bd9b8f7 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -67,11 +67,14 @@ class Program(log.Loggable): d = {} - for key in ['fat', 'special']: + for key, default in { + 'fat': True, + 'special': False + }.items(): value = None value = self._config.getboolean('main', 'path_filter_'+ key) if value is None: - value = True + value = default d[key] = value diff --git a/morituri/test/test_common_path.py b/morituri/test/test_common_path.py index 45b160d..7cec838 100644 --- a/morituri/test/test_common_path.py +++ b/morituri/test/test_common_path.py @@ -9,8 +9,17 @@ from morituri.test import common class FilterTestCase(common.TestCase): def setUp(self): - self._filter = path.PathFilter() + self._filter = path.PathFilter(special=True) def testSlash(self): part = u'A Charm/A Blade' self.assertEquals(self._filter.filter(part), u'A Charm-A Blade') + + def testFat(self): + part = u'A Word: F**k you?' + self.assertEquals(self._filter.filter(part), u'A Word- F__k you_') + + def testSpecial(self): + part = u'<<< $&*!\' "()`{}[]spaceship>>>' + self.assertEquals(self._filter.filter(part), + u'___ _____ ________spaceship___') diff --git a/morituri/test/test_program_cdparanoia.py b/morituri/test/test_program_cdparanoia.py index 4d7b510..6c910b1 100644 --- a/morituri/test/test_program_cdparanoia.py +++ b/morituri/test/test_program_cdparanoia.py @@ -69,7 +69,7 @@ class VersionTestCase(common.TestCase): self.failUnless(v) # of the form III 10.2 # make sure it ends with a digit - self.failUnless(int(v[-1])) + self.failUnless(int(v[-1]), v) class AnalyzeFileTask(cdparanoia.AnalyzeTask): From 948835a257d1c3e09832594e68e12af3b5844769 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 13 Jul 2013 23:03:46 +0200 Subject: [PATCH 039/132] More quote fixing --- morituri/common/path.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/morituri/common/path.py b/morituri/common/path.py index b7f39e8..8612e92 100644 --- a/morituri/common/path.py +++ b/morituri/common/path.py @@ -44,15 +44,24 @@ class PathFilter(object): if self._slashes: path = re.sub(r'[/\\]', '-', path, re.UNICODE) - if self._quotes: - path = re.sub(ur'[\u2019]', "'", path, re.UNICODE) - - if self._special: + def separators(path): # replace separators with a hyphen path = re.sub(r'[:\|]', '-', path, re.UNICODE) + return path + + # change all fancy single/double quotes to normal quotes + if self._quotes: + path = re.sub(ur'[\xc2\xb4\u2018\u2019\u201b]', "'", path, + re.UNICODE) + path = re.sub(ur'[\u201c\u201d\u201f]', '"', path, re.UNICODE) + + if self._special: + path = separators(path) path = re.sub(r'[\*\?&!\'\"\$\(\)`{}\[\]<>]', '_', path, re.UNICODE) if self._fat: + path = separators(path) + # : and | already gone, but leave them here for reference path = re.sub(r'[:\*\?"<>|"]', '_', path, re.UNICODE) return path From fdb8279234ab3730d35d2dfc304894c5b14aeb32 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 14 Jul 2013 09:05:58 +0200 Subject: [PATCH 040/132] make warning clearer. Fixes #7 --- morituri/rip/cd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 6b639ef..a0a29f7 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -237,7 +237,9 @@ Log files will log the path to tracks relative to this directory. if options.offset is None: options.offset = 0 - self.stdout.write("Using fallback read offset %d\n" % + self.stdout.write("""WARNING: using default offset %d. +Install pycdio and run 'rip offset find' to detect your drive's offset. +""" % options.offset) if self.options.output_directory is None: self.options.output_directory = os.getcwd() From 064c22e9e769b49f125367597383ded61d4ba354 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 14 Jul 2013 09:28:41 +0200 Subject: [PATCH 041/132] add test for directory module --- morituri/test/Makefile.am | 1 + morituri/test/test_common_directory.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 morituri/test/test_common_directory.py diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index 792beb4..eb3e13d 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -8,6 +8,7 @@ EXTRA_DIST = \ test_common_checksum.py \ test_common_common.py \ test_common_config.py \ + test_common_directory.py \ test_common_drive.py \ test_common_encode.py \ test_common_gstreamer.py \ diff --git a/morituri/test/test_common_directory.py b/morituri/test/test_common_directory.py new file mode 100644 index 0000000..c7b2766 --- /dev/null +++ b/morituri/test/test_common_directory.py @@ -0,0 +1,21 @@ +# -*- Mode: Python; test-case-name: morituri.test.test_common_directory -*- +# vi:si:et:sw=4:sts=4:ts=4 + +from morituri.common import directory + +from morituri.test import common + + +class DirectoryTestCase(common.TestCase): + + def testAll(self): + d = directory.Directory() + + path = d.getConfig() + self.failUnless(path.startswith('/home')) + + path = d.getCache() + self.failUnless(path.startswith('/home')) + + paths = d.getReadCaches() + self.failUnless(paths[0].startswith('/home')) From 02397302c6e7f79673533751839d5c6f39c9a289 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 14 Jul 2013 09:29:17 +0200 Subject: [PATCH 042/132] catch missing save_cache_path in xdg.BaseDirectory; closes #31 --- morituri/common/directory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/morituri/common/directory.py b/morituri/common/directory.py index e076a1f..47aac11 100644 --- a/morituri/common/directory.py +++ b/morituri/common/directory.py @@ -44,7 +44,8 @@ class Directory(log.Loggable): from xdg import BaseDirectory path = BaseDirectory.save_cache_path('morituri') self.info('Using XDG, cache directory is %s' % path) - except ImportError: + except (ImportError, AttributeError): + # save_cache_path was added in pyxdg 0.25 path = os.path.join(os.path.expanduser('~'), '.morituri', 'cache') if not os.path.exists(path): os.makedirs(path) @@ -65,7 +66,8 @@ class Directory(log.Loggable): path = BaseDirectory.save_cache_path('morituri') self.info('For XDG, read cache directory is %s' % path) paths.append(path) - except ImportError: + except (ImportError, AttributeError): + # save_cache_path was added in pyxdg 0.21 pass path = os.path.join(os.path.expanduser('~'), '.morituri', 'cache') From 888d3ffc2a29c590fbecf7aa7f986c83840c2716 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 14 Jul 2013 09:36:36 +0200 Subject: [PATCH 043/132] wait before reading, to make sure we get it all --- morituri/common/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/morituri/common/common.py b/morituri/common/common.py index 6608b7d..0af7186 100644 --- a/morituri/common/common.py +++ b/morituri/common/common.py @@ -319,6 +319,7 @@ class VersionGetter(object): p = asyncsub.Popen(self._args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + p.wait() output = asyncsub.recv_some(p, e=0, stderr=1) vre = self._regexp.search(output) if vre: From 365415a6c553a238bca453cd51faba92aea1072b Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 14 Jul 2013 09:55:35 +0200 Subject: [PATCH 044/132] Handle missing required argument to option. Fixes #18. --- morituri/extern/python-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/extern/python-command b/morituri/extern/python-command index f96c666..bea37f8 160000 --- a/morituri/extern/python-command +++ b/morituri/extern/python-command @@ -1 +1 @@ -Subproject commit f96c66672b0a674fb932562e1375f4e406f88f16 +Subproject commit bea37f88ecb02db5342e52d3ab0f61ec33d85b1f From a9a9795f86798260ac3b934213aeec4a91a88da8 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 14 Jul 2013 09:59:29 +0200 Subject: [PATCH 045/132] add bash-compgen. Fixes #11. --- etc/bash_completion.d/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/bash_completion.d/.gitignore b/etc/bash_completion.d/.gitignore index b6a9088..8981283 100644 --- a/etc/bash_completion.d/.gitignore +++ b/etc/bash_completion.d/.gitignore @@ -1 +1,2 @@ rip +bash-compgen From a19c73cb2eec74baaf992fbeced3b03713994ce7 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 15 Jul 2013 10:03:08 +0200 Subject: [PATCH 046/132] release 0.2.1 --- NEWS | 48 ++++++++++++++++++++++++++++++-- RELEASE | 77 ++++++++++++++++++++++++++++++++------------------- configure.ac | 2 +- morituri.doap | 22 +++++++++++++++ 4 files changed, 117 insertions(+), 32 deletions(-) diff --git a/NEWS b/NEWS index 216a932..0918e17 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,50 @@ -This is morituri 0.2.0, "ears" +This is morituri 0.2.1, "married" -Coverage in 0.2.0: 67 % (1890 / 2807), 95 python tests +Coverage in 0.2.1: 67 % (1957 / 2889), 101 python tests + +Features added in 0.2.1: + +- added "%X" template variable for uppercase filename extension +- added rip cd info +- added storing catalog number and barcode +- disambiguate releases with same name but different catno/barcode +- use all but last track to find offset +- add support to filter path names for better file system support +- add config options for path filtering +- fixes for older pyxdg and some versions of pycdio + +Bugs fixed in 0.2.1: + +in trac: +- 44: Optionally strip special characters from file names +- 121: ImportError: No module named CDDB +- 126: pycdio is no more optional : pkg_resources.DistributionNotFound: pycdio +- 135: rip drive analyze report "Cannot analyze the drive. Is there a CD in it?" when not able to defeat audio cache +- 137: pycdio returns an error when analyzing drive +- 138: Error when trying to rip with pycdio .19 +- 124: Checking of runtime dependencies + +in github: +- 31: Cryptic error message if xdg module is too old +- 30: AttributeError: Values instance has no attribute 'unknown' +- 26: Convert values returned from pycdio to str (workaround for upstream bug) +- 24: Filenames from musicbrainz may contain invalid characters for windows filesystems +- 23: Convert drive path from unicode to str when calling cdio.Device (pycdio 0.19 / Arch Linux) +- 22: Compare AccurateRip to num tracks -1, as last track not being checked +- 21: break up logger line +- 18: Crash if no path specified for '-O' option +- 17: Use XDG cache directory +- 16: Work with older versions of python-xdg +- 14: Use with statement to open files +- 13: Use os.path.join instead of hardcoded paths. +- 11: Ignore bash-compgen, to clean up git-status. +- 9: Ask which release to use if DiscID returns several matches +- 8: abort if invalid logger specified +- 7: Warn if no offset specified and no stored offset found/pycdio not available +- 6: Add "%X" template variable for uppercase filename extension. +- 3: (Optional) dependency on cddb should be documented +- 2: No module named moap.util -- dependency shoud be documented +- 1: No module named log -- use of submodules should be documented Features added in 0.2.0: diff --git a/RELEASE b/RELEASE index be74f73..88824f2 100644 --- a/RELEASE +++ b/RELEASE @@ -1,41 +1,60 @@ morituri is a CD ripper aiming for accuracy over speed. Its features are modeled to compare with Exact Audio Copy on Windows. -This is morituri 0.2.0 "ears". +This is morituri 0.2.1 "married". -This is intended as a release for daring and curious people who've had enough -of the fact that Windows has a more accurate CD ripper than Linux. +Coverage in 0.2.1: 67 % (1957 / 2889), 101 python tests -Coverage in 0.2.0: 67 % (1890 / 2807), 95 python tests +Features added in 0.2.1: -Features added in 0.2.0: +- added "%X" template variable for uppercase filename extension +- added rip cd info +- added storing catalog number and barcode +- disambiguate releases with same name but different catno/barcode +- use all but last track to find offset +- add support to filter path names for better file system support +- add config options for path filtering +- fixes for older pyxdg and some versions of pycdio -- added plugins system for logger -- added rip cd rip --logger to specify logger -- added reading speed, cdparanoia and cdrdao version to logger -- added rip drive analyze to detect whether we can defeat audio cache behaviour -- store drive offsets and cache defeating in config file -- rip drive list shows configured offset and audio cache defeating -- added rip image retag --release-id to specify the release id to tag with -- added %r/%R for release type to use in track/disc template -- added %x for extension to release template +Bugs fixed in 0.2.1: -Bugs fixed in 0.2.0: +in trac: +- 44: Optionally strip special characters from file names +- 121: ImportError: No module named CDDB +- 126: pycdio is no more optional : pkg_resources.DistributionNotFound: pycdio +- 135: rip drive analyze report "Cannot analyze the drive. Is there a CD in it?" when not able to defeat audio cache +- 137: pycdio returns an error when analyzing drive +- 138: Error when trying to rip with pycdio .19 +- 124: Checking of runtime dependencies -- 89: Fails to rip track with \ in its name -- 105: Backslash in track names causes "Cannot find file" during rip -- 108: Unable to find offset / rip -- 109: KeyError when running "rip offset find" -- 111: Python traceback when config has no read offset for CD -- 76: morituri should allow for a configuration file -- 96: rip image retag: allow specification of release ID -- 107: Backslash in track name confuses AR step -- 112: add MusicBrainz lookup URL to generated logfile +in github: +- 31: Cryptic error message if xdg module is too old +- 30: AttributeError: Values instance has no attribute 'unknown' +- 26: Convert values returned from pycdio to str (workaround for upstream bug) +- 24: Filenames from musicbrainz may contain invalid characters for windows filesystems +- 23: Convert drive path from unicode to str when calling cdio.Device (pycdio 0.19 / Arch Linux) +- 22: Compare AccurateRip to num tracks -1, as last track not being checked +- 21: break up logger line +- 18: Crash if no path specified for '-O' option +- 17: Use XDG cache directory +- 16: Work with older versions of python-xdg +- 14: Use with statement to open files +- 13: Use os.path.join instead of hardcoded paths. +- 11: Ignore bash-compgen, to clean up git-status. +- 9: Ask which release to use if DiscID returns several matches +- 8: abort if invalid logger specified +- 7: Warn if no offset specified and no stored offset found/pycdio not available +- 6: Add "%X" template variable for uppercase filename extension. +- 3: (Optional) dependency on cddb should be documented +- 2: No module named moap.util -- dependency shoud be documented +- 1: No module named log -- use of submodules should be documented -morituri 0.2.0 is brought to you by: -Loïc Minier -Ross Burton -Christophe Fergeau +morituri 0.2.1 is brought to you by: + Thomas Vander Stichele -mustbenice +Velo Superman +Nicolas Cornu +dioltas +Frederik "Freso" S. Olesen +Jonas Smedegaard diff --git a/configure.ac b/configure.ac index 886a527..a28ca7f 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl initialize autoconf dnl when going to/from release please remove/add the nano (fourth number) dnl releases only do Wall, trunk and prerelease does Werror too -AC_INIT(morituri, 0.2.0.1, +AC_INIT(morituri, 0.2.1, http://thomas.apestaart.org/morituri/trac/newticket, morituri) diff --git a/morituri.doap b/morituri.doap index 6e935a9..93d9525 100644 --- a/morituri.doap +++ b/morituri.doap @@ -40,6 +40,28 @@ Morituri is a CD ripper aiming for maximum quality. + + + 0.2.1 + master + married + 2013-07-14 + + + +- added "%X" template variable for uppercase filename extension +- added rip cd info +- added storing catalog number and barcode +- disambiguate releases with same name but different catno/barcode +- use all but last track to find offset +- add support to filter path names for better file system support +- add config options for path filtering +- fixes for older pyxdg and some versions of pycdio + + + + + 0.2.0 From 41a6640f08e4bd133c7053e17f0ec7c252ffe4b6 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 15 Jul 2013 10:03:45 +0200 Subject: [PATCH 047/132] back to development --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a28ca7f..97ce167 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl initialize autoconf dnl when going to/from release please remove/add the nano (fourth number) dnl releases only do Wall, trunk and prerelease does Werror too -AC_INIT(morituri, 0.2.1, +AC_INIT(morituri, 0.2.1.1, http://thomas.apestaart.org/morituri/trac/newticket, morituri) From a1fb4a6e706e6263ae2cdb581e7085eb0add3395 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 20 Jul 2013 13:21:11 +0200 Subject: [PATCH 048/132] remove done items --- TODO | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/TODO b/TODO index 6198f76..992568d 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,4 @@ TODO: -- add drive analysis mode - - use cdparanoia -A from 10.2 on for caching behaviour - store drive features in a database - try http://www.ime.usp.br/~pjssilva/secure-cdparanoia.py and see if it is better at handling some bad cd's @@ -16,13 +14,10 @@ TODO: or not putting the disk in) - check if it's simple to listen to each track in a multitrack completing - save trms to a pickle, after finishing each track -- add a way to store configuration data per drive, like offset - rip the data session - add AccurateRip validation for ripped images to rip command - add GUI -- persist RipResult so rips can be aborted and continued too; needs verification - of previously ripped files -- write moovida plugin +- write moovida/xbmc plugin - cache results of MusicBrainz lookups - on ana, Goldfrapp tells me I have offset 0! - don't keep short HTOA's if their peak level is low @@ -63,4 +58,3 @@ TODO: right now it is; maybe split in to base and output ? - rip task should abort on task 4 if checksums don't match - retry cdrdao a few times when it had to load the tray -- when it detects the target dir is already there, but the files would be different names, complain or customize the name with further info (see GLB - mockingbirds singles) From d52b008751d96051630510519f273cf1841263f1 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 21 Jul 2013 15:16:10 +0200 Subject: [PATCH 049/132] make sure we have a traceback --- morituri/common/checksum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/common/checksum.py b/morituri/common/checksum.py index 330af9d..e45550a 100644 --- a/morituri/common/checksum.py +++ b/morituri/common/checksum.py @@ -199,7 +199,7 @@ class ChecksumTask(log.Loggable, gstreamer.GstPipelineTask): msg = 'did not get all samples, %d of %d missing' % ( self._sampleEnd - last, self._sampleEnd) self.warning(msg) - self.setException(common.MissingFrames(msg)) + self.setExceptionAndTraceback(common.MissingFrames(msg)) return self.checksum = self._checksum From 697aafc21ddac81e566cef19205cdaba35e4f968 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 22 Jul 2013 01:54:00 +0200 Subject: [PATCH 050/132] handle exceptions on verifying empty or half-done tracks --- morituri/common/program.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index bd9b8f7..921b599 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -32,6 +32,8 @@ from morituri.common import common, log, mbngs, cache, path from morituri.program import cdrdao, cdparanoia from morituri.image import image +from morituri.extern.task import task, gstreamer + # FIXME: should Program have a runner ? @@ -522,7 +524,17 @@ class Program(log.Loggable): t = checksum.CRC32Task(trackResult.filename) - runner.run(t) + try: + runner.run(t) + except task.TaskException, e: + if isinstance(e.exception, common.MissingFrames): + self.warning('missing frames for %r' % trackResult.filename) + return False + elif isinstance(e.exception, gstreamer.GstException): + self.warning('GstException %r' % (e.exception, )) + return False + else: + raise ret = trackResult.testcrc == t.checksum log.debug('program', From 821dfcf6e22a831453235ad7343206e01ff2814c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 22 Jul 2013 01:54:20 +0200 Subject: [PATCH 051/132] tell us about each try --- morituri/rip/cd.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index a0a29f7..4ed098a 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -348,11 +348,14 @@ Install pycdio and run 'rip offset find' to detect your drive's offset. # we reset durations for test and copy here trackResult.testduration = 0.0 trackResult.copyduration = 0.0 - self.stdout.write('Ripping track %d of %d: %s\n' % ( - number, len(self.itable.tracks), - os.path.basename(path).encode('utf-8'))) + extra = "" while tries < MAX_TRIES: tries += 1 + if tries > 1: + extra = " (try %d)" % tries + self.stdout.write('Ripping track %d of %d%s: %s\n' % ( + number, len(self.itable.tracks), extra, + os.path.basename(path).encode('utf-8'))) try: self.debug('ripIfNotRipped: track %d, try %d', number, tries) @@ -361,8 +364,8 @@ Install pycdio and run 'rip offset find' to detect your drive's offset. device=self.parentCommand.options.device, profile=profile, taglist=self.program.getTagList(number), - what='track %d of %d' % ( - number, len(self.itable.tracks))) + what='track %d of %d%s' % ( + number, len(self.itable.tracks), extra)) break except Exception, e: self.debug('Got exception %r on try %d', From 5f7071e0b0e2bb8f44de8096e0363aaff5a719bb Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Thu, 25 Jul 2013 19:34:39 +0200 Subject: [PATCH 052/132] Fix rip offset find. Fixes #33. --- morituri/rip/offset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/rip/offset.py b/morituri/rip/offset.py index 26c14b2..ec88803 100644 --- a/morituri/rip/offset.py +++ b/morituri/rip/offset.py @@ -87,7 +87,7 @@ CD in the AccurateRip database.""" # this can be a symlink to another device def do(self, args): - prog = program.Program() + prog = program.Program(self.getRootCommand().config) runner = task.SyncRunner() device = self.options.device From 30e717b36bd56d9f19c2ba67e406ae8c4912e7ea Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 29 Jul 2013 11:40:43 +0200 Subject: [PATCH 053/132] use a logging task runner --- morituri/rip/cd.py | 3 +-- morituri/rip/offset.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 4ed098a..e927f62 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -28,12 +28,11 @@ import gobject gobject.threads_init() from morituri.common import logcommand, common, accurip, gstreamer -from morituri.common import drive, program +from morituri.common import drive, program, task from morituri.result import result from morituri.program import cdrdao, cdparanoia from morituri.rip import common as rcommon -from morituri.extern.task import task from morituri.extern.command import command diff --git a/morituri/rip/offset.py b/morituri/rip/offset.py index ec88803..646d46e 100644 --- a/morituri/rip/offset.py +++ b/morituri/rip/offset.py @@ -27,6 +27,7 @@ import gobject gobject.threads_init() from morituri.common import logcommand, accurip, drive, program +from morituri.common import task as ctask from morituri.program import cdrdao, cdparanoia from morituri.extern.task import task @@ -88,7 +89,7 @@ CD in the AccurateRip database.""" def do(self, args): prog = program.Program(self.getRootCommand().config) - runner = task.SyncRunner() + runner = ctask.SyncRunner() device = self.options.device From 430ee02a71d42b52b60c380a91408a22c413a0a8 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 29 Jul 2013 12:29:06 +0200 Subject: [PATCH 054/132] put space before colon --- morituri/common/path.py | 5 +++-- morituri/test/test_common_path.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/morituri/common/path.py b/morituri/common/path.py index 8612e92..cb595ae 100644 --- a/morituri/common/path.py +++ b/morituri/common/path.py @@ -45,8 +45,9 @@ class PathFilter(object): path = re.sub(r'[/\\]', '-', path, re.UNICODE) def separators(path): - # replace separators with a hyphen - path = re.sub(r'[:\|]', '-', path, re.UNICODE) + # replace separators with a space-hyphen or hyphen + path = re.sub(r'[:]', ' -', path, re.UNICODE) + path = re.sub(r'[\|]', '-', path, re.UNICODE) return path # change all fancy single/double quotes to normal quotes diff --git a/morituri/test/test_common_path.py b/morituri/test/test_common_path.py index 7cec838..84a66d3 100644 --- a/morituri/test/test_common_path.py +++ b/morituri/test/test_common_path.py @@ -17,9 +17,14 @@ class FilterTestCase(common.TestCase): def testFat(self): part = u'A Word: F**k you?' - self.assertEquals(self._filter.filter(part), u'A Word- F__k you_') + self.assertEquals(self._filter.filter(part), u'A Word - F__k you_') def testSpecial(self): part = u'<<< $&*!\' "()`{}[]spaceship>>>' self.assertEquals(self._filter.filter(part), u'___ _____ ________spaceship___') + + def testGreatest(self): + part = u'Greatest Ever! Soul: The Definitive Collection' + self.assertEquals(self._filter.filter(part), + u'Greatest Ever_ Soul - The Definitive Collection') From 85300a9529236a01a523d7699e49f7d4f26da5a8 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 29 Jul 2013 12:36:45 +0200 Subject: [PATCH 055/132] get revision from git; make it work everywhere --- configure.ac | 4 ++++ morituri/common/common.py | 15 +++++++++++++++ morituri/configure/configure.py | 2 -- morituri/configure/installed.py.in | 1 + morituri/configure/uninstalled.py.in | 5 ++++- 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 97ce167..f2e6445 100644 --- a/configure.ac +++ b/configure.ac @@ -38,6 +38,10 @@ AC_SUBST(PYTHONLIBDIR) AS_AC_EXPAND(PLUGINSDIR, "\${libdir}/morituri/plugins") AC_MSG_NOTICE(Setting plugins directory to $PLUGINSDIR) +dnl get git revision for installed.py.in +AC_SUBST(REVISION, `python -c 'from morituri.configure import configure; print configure.revision'`) +AC_MSG_NOTICE(Setting revision to $REVISION) + dnl check for epydoc AC_CHECK_PROG(EPYDOC, epydoc, yes, no) AM_CONDITIONAL(HAVE_EPYDOC, test "x$EPYDOC" = "xyes") diff --git a/morituri/common/common.py b/morituri/common/common.py index 0af7186..aa964e9 100644 --- a/morituri/common/common.py +++ b/morituri/common/common.py @@ -23,6 +23,7 @@ import os import os.path +import commands import math import subprocess @@ -331,3 +332,17 @@ class VersionGetter(object): raise return version + + +def getRevision(): + """ + Get a revision tag for the current git source tree. + + Appends -modified in case there are local modifications. + """ + describe = commands.getoutput('git describe') + + if commands.getoutput('git diff-index --name-only HEAD --'): + describe += '-modified' + + return describe diff --git a/morituri/configure/configure.py b/morituri/configure/configure.py index f55525a..6a38329 100644 --- a/morituri/configure/configure.py +++ b/morituri/configure/configure.py @@ -22,8 +22,6 @@ import os # where am I on the disk ? __thisdir = os.path.dirname(os.path.abspath(__file__)) -revision = "$Revision$" - if os.path.exists(os.path.join(__thisdir, 'uninstalled.py')): from morituri.configure import uninstalled config_dict = uninstalled.get() diff --git a/morituri/configure/installed.py.in b/morituri/configure/installed.py.in index c2c536a..c29a3f5 100644 --- a/morituri/configure/installed.py.in +++ b/morituri/configure/installed.py.in @@ -7,4 +7,5 @@ def get(): 'isinstalled': True, 'pluginsdir': '@PLUGINSDIR@', 'version': '@VERSION@', + 'revision', '@REVISION@', } diff --git a/morituri/configure/uninstalled.py.in b/morituri/configure/uninstalled.py.in index 7e9aea3..a8155f8 100644 --- a/morituri/configure/uninstalled.py.in +++ b/morituri/configure/uninstalled.py.in @@ -1,7 +1,9 @@ # -*- Mode: Python -*- # vi:si:et:sw=4:sts=4:ts=4 -import os +import os.path + +from morituri.common import common __thisdir = os.path.dirname(os.path.abspath(__file__)) @@ -13,4 +15,5 @@ def get(): 'pluginsdir': os.path.abspath(os.path.join( __thisdir, '..', '..', 'plugins')), 'version': '@VERSION@', + 'revision': common.getRevision(), } From 28926e3fba5b88886631e73a472476558b0c6998 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 10:59:28 +0200 Subject: [PATCH 056/132] handle not having cdparanoia on rip offset find --- morituri/rip/offset.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/morituri/rip/offset.py b/morituri/rip/offset.py index 646d46e..965c9f0 100644 --- a/morituri/rip/offset.py +++ b/morituri/rip/offset.py @@ -26,7 +26,7 @@ import tempfile import gobject gobject.threads_init() -from morituri.common import logcommand, accurip, drive, program +from morituri.common import logcommand, accurip, drive, program, common from morituri.common import task as ctask from morituri.program import cdrdao, cdparanoia @@ -151,11 +151,18 @@ CD in the AccurateRip database.""" try: archecksum = self._arcs(runner, table, 1, offset) except task.TaskException, e: + + # let MissingDependency fall through + if isinstance(e.exception, + common.MissingDependencyException): + raise e + if isinstance(e.exception, cdparanoia.FileSizeError): self.stdout.write( 'WARNING: cannot rip with offset %d...\n' % offset) continue - self.warning("Unknown exception for offset %d: %r" % ( + + self.warning("Unknown task exception for offset %d: %r" % ( offset, e)) self.stdout.write( 'WARNING: cannot rip with offset %d...\n' % offset) From fa6cb73c55fe69ee9cc0172b6231c637b10b67db Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 11:00:30 +0200 Subject: [PATCH 057/132] update --- doc/release | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release b/doc/release index 9590a3a..f0fd883 100644 --- a/doc/release +++ b/doc/release @@ -16,6 +16,7 @@ pre-release checklist - Verify the program runs: - normal run - --unknown run + - rip offset find - add new milestone to trac and make it the default - verify with ticket query that all fixed tickets for this milestone are correct: From 6aff8296bf25b26679bd5754dcf45de2465581c6 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 11:32:41 +0200 Subject: [PATCH 058/132] Make rip cd info not eject. Fixes #35. --- morituri/rip/cd.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index e927f62..c2c1a94 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -43,8 +43,11 @@ class _CD(logcommand.LogCommand): """ @type program: L{program.Program} + @ivar eject: whether to eject the drive after completing """ + eject = True + def addOptions(self): # FIXME: have a cache of these pickles somewhere self.parser.add_option('-T', '--toc-pickle', @@ -94,7 +97,7 @@ class _CD(logcommand.LogCommand): self.stdout.write('FreeDB identifies disc as %s\n' % cddbmd) # also used by rip cd info - if not getattr(self.options, 'unknown', False): + if not getattr(self.options, 'unknown', False) and self.eject: self.program.ejectDevice(self.device) return -1 @@ -150,7 +153,8 @@ class _CD(logcommand.LogCommand): self.doCommand() - self.program.ejectDevice(self.device) + if self.eject: + self.program.ejectDevice(self.device) def doCommand(self): pass @@ -159,6 +163,8 @@ class _CD(logcommand.LogCommand): class Info(_CD): summary = "retrieve information about the currently inserted CD" + eject = False + class Rip(_CD): summary = "rip CD" From d88d9f9bc66dbcca4f2ee6d1f1e764fd8eb5964e Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 11:39:40 +0200 Subject: [PATCH 059/132] rename metadata to discMD to be clearer --- morituri/common/mbngs.py | 43 +++++++++++++++++++------------------- morituri/common/program.py | 2 +- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index 055899c..aaf9ca1 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -60,6 +60,7 @@ class DiscMetadata(object): @type release: unicode @param title: title of the disc (with disambiguation) @param releaseTitle: title of the release (without disambiguation) + @type tracks: C{list} of L{TrackMetadata} """ artist = None sortName = None @@ -109,9 +110,9 @@ def _getMetadata(releaseShort, release, discid): assert release['id'], 'Release does not have an id' - metadata = DiscMetadata() + discMD = DiscMetadata() - metadata.releaseType = releaseShort.get('release-group', {}).get('type') + discMD.releaseType = releaseShort.get('release-group', {}).get('type') credit = release['artist-credit'] artist = credit[0]['artist'] @@ -127,27 +128,27 @@ def _getMetadata(releaseShort, release, discid): albumArtistName = "".join(credit) # FIXME: is there a better way to check for VA - metadata.various = False + discMD.various = False if artist['id'] == VA_ID: - metadata.various = True + discMD.various = True # getUniqueName gets disambiguating names like Muse (UK rock band) - metadata.artist = albumArtistName - metadata.sortName = artist['sort-name'] + discMD.artist = albumArtistName + discMD.sortName = artist['sort-name'] # FIXME: is format str ? if not 'date' in release: log.warning('mbngs', 'Release %r does not have date', release) else: - metadata.release = release['date'] + discMD.release = release['date'] - metadata.mbid = release['id'] - metadata.mbidArtist = artist['id'] - metadata.url = 'http://musicbrainz.org/release/' + release['id'] + discMD.mbid = release['id'] + discMD.mbidArtist = artist['id'] + discMD.url = 'http://musicbrainz.org/release/' + release['id'] - metadata.barcode = release.get('barcode', None) + discMD.barcode = release.get('barcode', None) lil = release.get('label-info-list', [{}]) if lil: - metadata.catalogNumber = lil[0].get('catalog-number') + discMD.catalogNumber = lil[0].get('catalog-number') tainted = False duration = 0 @@ -156,7 +157,7 @@ def _getMetadata(releaseShort, release, discid): for disc in medium['disc-list']: if disc['id'] == discid: title = release['title'] - metadata.releaseTitle = title + discMD.releaseTitle = title if 'disambiguation' in release: title += " (%s)" % release['disambiguation'] count = len(release['medium-list']) @@ -165,7 +166,7 @@ def _getMetadata(releaseShort, release, discid): int(medium['position']), count) if 'title' in medium: title += ": %s" % medium['title'] - metadata.title = title + discMD.title = title for t in medium['track-list']: track = TrackMetadata() credit = t['recording']['artist-credit'] @@ -182,9 +183,9 @@ def _getMetadata(releaseShort, release, discid): trackArtistName = "".join(credit) if not artist: - track.artist = metadata.artist - track.sortName = metadata.sortName - track.mbidArtist = metadata.mbidArtist + track.artist = discMD.artist + track.sortName = discMD.sortName + track.mbidArtist = discMD.mbidArtist else: # various artists discs can have tracks with no artist track.artist = trackArtistName @@ -204,14 +205,14 @@ def _getMetadata(releaseShort, release, discid): else: duration += track.duration - metadata.tracks.append(track) + discMD.tracks.append(track) if not tainted: - metadata.duration = duration + discMD.duration = duration else: - metadata.duration = 0 + discMD.duration = 0 - return metadata + return discMD # see http://bugs.musicbrainz.org/browser/python-musicbrainz2/trunk/examples/ diff --git a/morituri/common/program.py b/morituri/common/program.py index 921b599..4bdced7 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -43,7 +43,7 @@ class Program(log.Loggable): I maintain program state and functionality. @ivar metadata: - @type metadata: L{musicbrainz.DiscMetadata} + @type metadata: L{mbngs.DiscMetadata} @ivar result: the rip's result @type result: L{result.RipResult} @type outdir: unicode From 6db23603ab59c4210d4aaa425c6839cb5ce55c72 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 12:34:33 +0200 Subject: [PATCH 060/132] rename to discArtist to disambiguate --- morituri/common/mbngs.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index aaf9ca1..c140a27 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -114,8 +114,12 @@ def _getMetadata(releaseShort, release, discid): discMD.releaseType = releaseShort.get('release-group', {}).get('type') credit = release['artist-credit'] + # example: + # [{'artist': + # {'sort-name': 'Pixies', + # 'id': 'b6b2bb8d-54a9-491f-9607-7b546023b433', 'name': 'Pixies'}}] - artist = credit[0]['artist'] + discArtist = credit[0]['artist'] if len(credit) > 1: log.debug('mbngs', 'artist-credit more than 1: %r', credit) @@ -129,12 +133,12 @@ def _getMetadata(releaseShort, release, discid): # FIXME: is there a better way to check for VA discMD.various = False - if artist['id'] == VA_ID: + if discArtist['id'] == VA_ID: discMD.various = True # getUniqueName gets disambiguating names like Muse (UK rock band) discMD.artist = albumArtistName - discMD.sortName = artist['sort-name'] + discMD.sortName = discArtist['sort-name'] # FIXME: is format str ? if not 'date' in release: log.warning('mbngs', 'Release %r does not have date', release) @@ -142,7 +146,7 @@ def _getMetadata(releaseShort, release, discid): discMD.release = release['date'] discMD.mbid = release['id'] - discMD.mbidArtist = artist['id'] + discMD.mbidArtist = discArtist['id'] discMD.url = 'http://musicbrainz.org/release/' + release['id'] discMD.barcode = release.get('barcode', None) @@ -174,7 +178,18 @@ def _getMetadata(releaseShort, release, discid): log.debug('mbngs', 'artist-credit more than 1: %r', credit) # credit is of the form [dict, str, dict, ... ] + # e.g. [ + # {'artist': { + # 'sort-name': 'Sukilove', + # 'id': '5f4af6cf-a1b8-4e51-a811-befed399a1c6', + # 'name': 'Sukilove' + # }}, ' & ', { + # 'artist': { + # 'sort-name': 'Blackie and the Oohoos', + # 'id': '028a9dc7-f5ef-43c2-866b-08d69ffff363', + # 'name': 'Blackie & the Oohoos'}}] for i, c in enumerate(credit): + # replace dict with the artist name if isinstance(c, dict): credit[i] = c.get( 'name', c['artist'].get('name', None)) @@ -182,15 +197,15 @@ def _getMetadata(releaseShort, release, discid): trackArtistName = "".join(credit) - if not artist: + if not discArtist: track.artist = discMD.artist track.sortName = discMD.sortName track.mbidArtist = discMD.mbidArtist else: # various artists discs can have tracks with no artist track.artist = trackArtistName - track.sortName = artist['sort-name'] - track.mbidArtist = artist['id'] + track.sortName = discArtist['sort-name'] + track.mbidArtist = discArtist['id'] track.title = t['recording']['title'] track.mbid = t['recording']['id'] From 6c842eb19e059f3e9fcf3252482fc674a524f5ec Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 12:34:50 +0200 Subject: [PATCH 061/132] add doc; make sure we record with rip debug musicbrainzngs --- morituri/rip/debug.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index e182109..369be4c 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -214,6 +214,8 @@ class MusicBrainzNGS(logcommand.LogCommand): summary = "examine MusicBrainz NGS info" description = """Look up a MusicBrainz disc id and output information. +You can get the MusicBrainz disc id with rip cd info. + Example disc id: KnpGsLhvH.lPrNc1PBL21lb9Bg4-""" def do(self, args): @@ -224,7 +226,8 @@ Example disc id: KnpGsLhvH.lPrNc1PBL21lb9Bg4-""" return 3 from morituri.common import mbngs - metadatas = mbngs.musicbrainz(discId) + metadatas = mbngs.musicbrainz(discId, + record=self.getRootCommand().record) self.stdout.write('%d releases\n' % len(metadatas)) for i, md in enumerate(metadatas): From 321f871805803ab44d6e1d01bb2e13eabb1d6ac9 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 12:35:23 +0200 Subject: [PATCH 062/132] add a test case to fix --- morituri/test/Makefile.am | 1 + ....a76714e0-32b1-4ed4-b28e-f86d99642193.json | 1 + morituri/test/test_common_mbngs.py | 27 +++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 morituri/test/morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index eb3e13d..d5ac917 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -37,6 +37,7 @@ EXTRA_DIST = \ release.08397059-86c1-463b-8ed0-cd596dbd174f.xml \ release.93a6268c-ddf1-4898-bf93-fb862b1c5c5e.xml \ release.c7d919f4-3ea0-4c4b-a230-b3605f069440.xml \ + morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json \ morituri.release.3451f29c-9bb8-4cc5-bfcc-bd50104b94f8.json \ kanye.cue \ kings-separate.cue \ diff --git a/morituri/test/morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json b/morituri/test/morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json new file mode 100644 index 0000000..b7390a9 --- /dev/null +++ b/morituri/test/morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json @@ -0,0 +1 @@ +{"release": {"status": "Official", "artist-credit": [{"artist": {"sort-name": "Various Artists", "id": "89ad4ac3-39f7-470e-963a-56509c546377", "name": "Various Artists"}}], "title": "2 Meter Sessies, Volume 10", "label-info-list": [], "medium-list": [{"disc-list": [{"id": "f7XO36a7n1LCCskkCiulReWbwZA-", "sectors": "317128"}], "position": "1", "track-list": [{"recording": {"artist-credit": [{"artist": {"sort-name": "Coldplay", "id": "cc197bad-dc9c-440d-a5b5-d52ba2e14234", "name": "Coldplay"}}], "length": "265200", "artist-credit-phrase": "Coldplay", "id": "06813123-5047-4c94-88bf-6a300540e954", "title": "Trouble"}, "position": "1"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Live", "id": "cba77ba2-862d-4cee-a8f6-d3f9daf7211c", "name": "Live"}}], "length": "264466", "artist-credit-phrase": "Live", "id": "7b57b108-35bb-4fcb-9046-06228fb7e5f7", "title": "Run to the Water"}, "position": "2"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Beck", "id": "309c62ba-7a22-4277-9f67-4a162526d18a", "name": "Beck"}}], "length": "384440", "artist-credit-phrase": "Beck", "id": "cfbfb04e-ccfd-4316-a5eb-5e4daa670905", "title": "Debra"}, "position": "3"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Jayhawks, The", "id": "24ed5b09-02b1-47fe-bd83-6fa5270039b0", "name": "The Jayhawks"}}], "length": "261746", "artist-credit-phrase": "The Jayhawks", "id": "d7b84a3f-628d-49f3-ae2f-b34d5630ee45", "title": "Mr. Wilson"}, "position": "4"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Dijk, De", "id": "d7a55e92-a14c-4543-8152-de2163af06bb", "name": "De Dijk"}}], "length": "239080", "artist-credit-phrase": "De Dijk", "id": "2bb1a0ca-399a-4488-8a53-bb0ca9ece5ea", "title": "Wie het niet weet"}, "position": "5"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Torrini, Emil\u00edana", "id": "b2a9731b-9e13-4ff9-af21-5e694a5663e8", "name": "Emil\u00edana Torrini"}}], "length": "209800", "artist-credit-phrase": "Emil\u00edana Torrini", "id": "3e6433da-9af5-41b0-9210-6dbef13c630c", "title": "Summer Breeze"}, "position": "6"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Hiatt, John", "id": "e78202c9-7717-435c-9aac-dd5ebc4e64d5", "name": "John Hiatt"}}], "length": "159600", "artist-credit-phrase": "John Hiatt", "id": "ce684ade-741f-47f7-ac1c-0d7d206da8a3", "title": "What Do We Do Now"}, "position": "7"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Hay, Barry & Barking Dogs", "id": "dc11b420-0e21-4e05-aca7-273f58c8bcce", "name": "Barry Hay & Barking Dogs"}}], "length": "184800", "artist-credit-phrase": "Barry Hay & Barking Dogs", "id": "56b1b506-64cd-4c2f-be6d-044e3888dabd", "title": "Happiness Is a Warm Gun"}, "position": "8"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Penn, Dan", "id": "cc54ec8d-ba66-4051-970d-6b3c24cd9e8b", "name": "Dan Penn"}}, " & ", {"artist": {"sort-name": "Oldham, Spooner", "id": "ba170eca-541b-4ee5-b332-54ff954b75ea", "name": "Spooner Oldham"}}], "length": "231640", "artist-credit-phrase": "Dan Penn & Spooner Oldham", "id": "4d1c6a29-dd96-4af6-a594-622d70e214ac", "title": "I'm Your Puppet"}, "position": "9"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Stone, Angie", "id": "82f8dd22-0319-4f35-953c-358b3f883027", "name": "Angie Stone"}}], "length": "211800", "artist-credit-phrase": "Angie Stone", "id": "cb0447fc-3ad3-4dea-a94d-517179a6d68c", "title": "Everyday"}, "position": "10"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Helsen, Tom", "id": "0a5fe43b-ace7-407b-bfc2-be4851e7d3f2", "name": "Tom Helsen"}}], "length": "249693", "artist-credit-phrase": "Tom Helsen", "id": "2f6501f8-262a-4f02-a782-ed365621e100", "title": "When Marvin Calls"}, "position": "11"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "K's Choice", "id": "9bd1e632-b17b-4842-b520-ddfce3b538b9", "name": "K\u2019s Choice"}}], "length": "210666", "artist-credit-phrase": "K\u2019s Choice", "id": "e3ef3fa1-3155-464d-a5e0-4096e9cc63ad", "title": "Almost Happy"}, "position": "12"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Casey, Paddy", "id": "d36a3897-f76d-4227-be80-d0d7282ff12a", "name": "Paddy Casey"}}], "length": "191000", "artist-credit-phrase": "Paddy Casey", "id": "c419e7a6-cbe7-44c9-a45e-08e0721695dd", "title": "Can't Take That Away"}, "position": "13"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Jackson, Joe", "id": "07f6d469-38f3-46da-9cfa-2f532422b84e", "name": "Joe Jackson"}}], "length": "267933", "artist-credit-phrase": "Joe Jackson", "id": "ebb7083f-4db2-4daa-a67d-2993887b67ad", "title": "Stranger Than You"}, "position": "14"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "My Morning Jacket", "id": "ea5883b7-68ce-48b3-b115-61746ea53b8c", "name": "My Morning Jacket"}}], "length": "325466", "artist-credit-phrase": "My Morning Jacket", "id": "62594b12-5907-42b6-b7d9-03ad5b0ddd35", "title": "Old September Blues"}, "position": "15"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Jones, Tom", "id": "57c6f649-6cde-48a7-8114-2a200247601a", "name": "Tom Jones"}}, " & ", {"artist": {"sort-name": "Stereophonics", "id": "0bfba3d3-6a04-4779-bb0a-df07df5b0558", "name": "Stereophonics"}}], "length": "193973", "artist-credit-phrase": "Tom Jones & Stereophonics", "id": "ba50a1c7-9e23-4c3e-b7aa-12e23eea6d19", "title": "Mama Told Me Not to Come"}, "position": "16"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Christophers, Ben", "id": "1a5b4ad0-593a-4069-a77d-dae722a5f0ac", "name": "Ben Christophers"}}], "length": "223333", "artist-credit-phrase": "Ben Christophers", "id": "c0cfc4cb-8c80-4516-b500-2df010418697", "title": "Sunday"}, "position": "17"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Barman, Tom", "id": "a9be8bc0-47a4-4a0b-af5f-feac18d3bc43", "name": "Tom Barman"}}, " & ", {"artist": {"sort-name": "Nueten, Van, Guy", "id": "8779d2fd-3fc8-4c1e-a37d-2edf66b07c4e", "name": "Guy Van Nueten"}}], "length": "151733", "artist-credit-phrase": "Tom Barman & Guy Van Nueten", "id": "e423a1d7-3ae1-4540-b267-d873c50043e7", "title": "Magnolia"}, "position": "18"}], "format": "CD"}], "text-representation": {"language": "eng", "script": "Latn"}, "date": "2001-10-15", "artist-credit-phrase": "Various Artists", "quality": "normal", "id": "a76714e0-32b1-4ed4-b28e-f86d99642193"}} \ No newline at end of file diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index ba5f30b..f18f3fa 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -11,6 +11,7 @@ from morituri.common import mbngs class MetadataTestCase(unittest.TestCase): + # Generated with rip -R cd info def testJeffEverybodySingle(self): path = os.path.join(os.path.dirname(__file__), 'morituri.release.3451f29c-9bb8-4cc5-bfcc-bd50104b94f8.json') @@ -22,3 +23,29 @@ class MetadataTestCase(unittest.TestCase): metadata = mbngs._getMetadata({}, response['release'], discid) self.failIf(metadata.release) + + def test2MeterSessies10(self): + # various artists, multiple artists per track + path = os.path.join(os.path.dirname(__file__), + 'morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json') + handle = open(path, "rb") + response = json.loads(handle.read()) + handle.close() + discid = "f7XO36a7n1LCCskkCiulReWbwZA-" + + metadata = mbngs._getMetadata({}, response['release'], discid) + + self.assertEquals(metadata.artist, u'Various Artists') + self.assertEquals(metadata.release, u'2001-10-15') + self.assertEquals(metadata.mbidArtist, + u'89ad4ac3-39f7-470e-963a-56509c546377') + + self.assertEquals(len(metadata.tracks), 18) + + track16 = metadata.tracks[15] + + self.assertEquals(track16.artist, 'Tom Jones & Stereophonics') + # FIXME: this is the disc artist id, and it should be the combo + # of track artist id's + self.assertEquals(track16.mbidArtist, + u'89ad4ac3-39f7-470e-963a-56509c546377') From 82404900fd8e7510cdaff6a561082dd2f630679f Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 12:36:21 +0200 Subject: [PATCH 063/132] fix syntax error --- morituri/configure/installed.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/configure/installed.py.in b/morituri/configure/installed.py.in index c29a3f5..c80f431 100644 --- a/morituri/configure/installed.py.in +++ b/morituri/configure/installed.py.in @@ -7,5 +7,5 @@ def get(): 'isinstalled': True, 'pluginsdir': '@PLUGINSDIR@', 'version': '@VERSION@', - 'revision', '@REVISION@', + 'revision': '@REVISION@', } From 7533b269fc8f530f187fdc0472df9e7bab4e95bc Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 12:47:00 +0200 Subject: [PATCH 064/132] add another test to fix --- morituri/test/Makefile.am | 1 + ....e32ae79a-336e-4d33-945c-8c5e8206dbd3.json | 1 + morituri/test/test_common_mbngs.py | 26 +++++++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 morituri/test/morituri.release.e32ae79a-336e-4d33-945c-8c5e8206dbd3.json diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index d5ac917..1b5b8d4 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -39,6 +39,7 @@ EXTRA_DIST = \ release.c7d919f4-3ea0-4c4b-a230-b3605f069440.xml \ morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json \ morituri.release.3451f29c-9bb8-4cc5-bfcc-bd50104b94f8.json \ + morituri.release.e32ae79a-336e-4d33-945c-8c5e8206dbd3.json \ kanye.cue \ kings-separate.cue \ kings-single.cue \ diff --git a/morituri/test/morituri.release.e32ae79a-336e-4d33-945c-8c5e8206dbd3.json b/morituri/test/morituri.release.e32ae79a-336e-4d33-945c-8c5e8206dbd3.json new file mode 100644 index 0000000..e5bb887 --- /dev/null +++ b/morituri/test/morituri.release.e32ae79a-336e-4d33-945c-8c5e8206dbd3.json @@ -0,0 +1 @@ +{"release": {"status": "Official", "asin": "B000CNEQ64", "label-info-list": [{"label": {"sort-name": "V2 Records International", "id": "947c12a1-cf28-4380-a695-a944ad15e387", "name": "V2 Records International"}, "catalog-number": "VVR1035822"}], "title": "Ballad of the Broken Seas", "country": "GB", "barcode": "5033197358222", "artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "medium-list": [{"disc-list": [{"id": "xAq8L4ELMW14.6wI6tt7QAcxiDI-", "sectors": "192868"}], "position": "1", "track-list": [{"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "171613", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "4fe44724-1d7e-4275-9693-b889864de750", "title": "Deus Ibi Est"}, "position": "1"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "190120", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "32047729-7ad9-42ae-8d9e-c256ef9251ec", "title": "Black Mountain"}, "position": "2"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "233880", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "0c71631a-5862-4834-ae8f-257b64bca745", "title": "The False Husband"}, "position": "3"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "162386", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "afc9e785-60fd-4942-a23c-3653633f4783", "title": "Ballad of the Broken Seas"}, "position": "4"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "160680", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "048932de-992d-4b08-ab4f-b5d735ea323e", "title": "Revolver"}, "position": "5"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "209066", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "42c0e096-6c48-43cf-b6d4-700903727418", "title": "Ramblin' Man"}, "position": "6"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "207133", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "ef599a4c-8163-4829-9332-8dfe8c79219a", "title": "(Do You Wanna) Come Walk With Me?"}, "position": "7"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "277186", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "765fc7cc-2055-4066-a5b2-f1afbd1fd1f8", "title": "Saturday's Gone"}, "position": "8"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "173640", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "61ac7fad-d396-4467-93a9-a25472561008", "title": "It's Hard to Kill a Bad Thing"}, "position": "9"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "224173", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "2fed65ae-3297-40d6-8f54-0d55f8ed7287", "title": "Honey Child What Can I Do?"}, "position": "10"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "224560", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "33ce6721-b148-45ad-9a1e-1a4b1ea6912e", "title": "Dusty Wreath"}, "position": "11"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Campbell, Isobel", "id": "d51f3a15-12a2-41a0-acfa-33b5eae71164", "name": "Isobel Campbell"}}, " & ", {"artist": {"sort-name": "Lanegan, Mark", "id": "a9126556-f555-4920-9617-6e013f8228a7", "name": "Mark Lanegan"}}], "length": "335133", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "id": "6cdb184d-12a0-4ba8-b50b-3325e0664f9e", "title": "The Circus Is Leaving Town"}, "position": "12"}], "format": "CD"}], "text-representation": {"language": "eng", "script": "Latn"}, "date": "2006-01-30", "artist-credit-phrase": "Isobel Campbell & Mark Lanegan", "quality": "normal", "id": "e32ae79a-336e-4d33-945c-8c5e8206dbd3"}} \ No newline at end of file diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index f18f3fa..964049d 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -49,3 +49,29 @@ class MetadataTestCase(unittest.TestCase): # of track artist id's self.assertEquals(track16.mbidArtist, u'89ad4ac3-39f7-470e-963a-56509c546377') + + def testBalladOfTheBrokenSeas(self): + # various artists disc + path = os.path.join(os.path.dirname(__file__), + 'morituri.release.e32ae79a-336e-4d33-945c-8c5e8206dbd3.json') + handle = open(path, "rb") + response = json.loads(handle.read()) + handle.close() + discid = "xAq8L4ELMW14.6wI6tt7QAcxiDI-" + + metadata = mbngs._getMetadata({}, response['release'], discid) + + self.assertEquals(metadata.artist, u'Isobel Campbell & Mark Lanegan') + self.assertEquals(metadata.release, u'2006-01-30') + # FIXME: this is only Isobel + self.assertEquals(metadata.mbidArtist, + u'd51f3a15-12a2-41a0-acfa-33b5eae71164') + + self.assertEquals(len(metadata.tracks), 12) + + track12 = metadata.tracks[11] + + self.assertEquals(track12.artist, u'Isobel Campbell & Mark Lanegan') + # FIXME: this is only Isobel + self.assertEquals(track12.mbidArtist, + u'd51f3a15-12a2-41a0-acfa-33b5eae71164') From 7b179556643d33a2845a56c7b3195caae2317311 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:05:09 +0200 Subject: [PATCH 065/132] add doc --- morituri/common/mbngs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index c140a27..100dca9 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -56,6 +56,8 @@ class TrackMetadata(object): class DiscMetadata(object): """ + @param artist: artist(s) name + @param sortName: album artist sort name @param release: earliest release date, in YYYY-MM-DD @type release: unicode @param title: title of the disc (with disambiguation) From 3d53056573d18126010faf41bbac064ed024b276 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:05:17 +0200 Subject: [PATCH 066/132] Extract a Credit list subclass to get name/sortname/id --- morituri/common/mbngs.py | 67 ++++++++++++++++++------------ morituri/test/test_common_mbngs.py | 2 + 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index 100dca9..9543e8b 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -95,6 +95,43 @@ def _record(record, which, name, what): handle.close() log.info('mbngs', 'Wrote %s %s to %s', which, name, filename) +# credit is of the form [dict, str, dict, ... ] +# e.g. [ +# {'artist': { +# 'sort-name': 'Sukilove', +# 'id': '5f4af6cf-a1b8-4e51-a811-befed399a1c6', +# 'name': 'Sukilove' +# }}, ' & ', { +# 'artist': { +# 'sort-name': 'Blackie and the Oohoos', +# 'id': '028a9dc7-f5ef-43c2-866b-08d69ffff363', +# 'name': 'Blackie & the Oohoos'}}] + + +class _Credit(list): + """ + I am a representation of an artist-credit in musicbrainz for a disc + or track. + """ + + def joiner(self, attributeGetter): + res = [] + + for item in self: + if isinstance(item, dict): + res.append(attributeGetter(item)) + else: + res.append(item) + + return "".join(res) + + + def getSortName(self): + return self.joiner(lambda i: i.get('sort-name')) + + def getName(self): + return self.joiner(lambda i: i.get('artist').get('name', None)) + def _getMetadata(releaseShort, release, discid): """ @@ -115,7 +152,7 @@ def _getMetadata(releaseShort, release, discid): discMD = DiscMetadata() discMD.releaseType = releaseShort.get('release-group', {}).get('type') - credit = release['artist-credit'] + credit = _Credit(release['artist-credit']) # example: # [{'artist': # {'sort-name': 'Pixies', @@ -126,12 +163,7 @@ def _getMetadata(releaseShort, release, discid): if len(credit) > 1: log.debug('mbngs', 'artist-credit more than 1: %r', credit) - for i, c in enumerate(credit): - if isinstance(c, dict): - credit[i] = c.get( - 'name', c['artist'].get('name', None)) - - albumArtistName = "".join(credit) + albumArtistName = credit.getName() # FIXME: is there a better way to check for VA discMD.various = False @@ -175,29 +207,12 @@ def _getMetadata(releaseShort, release, discid): discMD.title = title for t in medium['track-list']: track = TrackMetadata() - credit = t['recording']['artist-credit'] + credit = _Credit(t['recording']['artist-credit']) if len(credit) > 1: log.debug('mbngs', 'artist-credit more than 1: %r', credit) - # credit is of the form [dict, str, dict, ... ] - # e.g. [ - # {'artist': { - # 'sort-name': 'Sukilove', - # 'id': '5f4af6cf-a1b8-4e51-a811-befed399a1c6', - # 'name': 'Sukilove' - # }}, ' & ', { - # 'artist': { - # 'sort-name': 'Blackie and the Oohoos', - # 'id': '028a9dc7-f5ef-43c2-866b-08d69ffff363', - # 'name': 'Blackie & the Oohoos'}}] - for i, c in enumerate(credit): - # replace dict with the artist name - if isinstance(c, dict): - credit[i] = c.get( - 'name', c['artist'].get('name', None)) - - trackArtistName = "".join(credit) + trackArtistName = credit.getName() if not discArtist: track.artist = discMD.artist diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index 964049d..a06b958 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -62,6 +62,8 @@ class MetadataTestCase(unittest.TestCase): metadata = mbngs._getMetadata({}, response['release'], discid) self.assertEquals(metadata.artist, u'Isobel Campbell & Mark Lanegan') + # FIXME: this should include Mark + self.assertEquals(metadata.sortName, u'Campbell, Isobel') self.assertEquals(metadata.release, u'2006-01-30') # FIXME: this is only Isobel self.assertEquals(metadata.mbidArtist, From a8ef4a54c46b45db848676b76b5e8c93407f211d Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:07:09 +0200 Subject: [PATCH 067/132] make sure sortname contains all artists --- morituri/common/mbngs.py | 4 ++-- morituri/test/test_common_mbngs.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index 9543e8b..1fca73e 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -127,7 +127,7 @@ class _Credit(list): def getSortName(self): - return self.joiner(lambda i: i.get('sort-name')) + return self.joiner(lambda i: i.get('artist').get('sort-name', None)) def getName(self): return self.joiner(lambda i: i.get('artist').get('name', None)) @@ -172,7 +172,7 @@ def _getMetadata(releaseShort, release, discid): # getUniqueName gets disambiguating names like Muse (UK rock band) discMD.artist = albumArtistName - discMD.sortName = discArtist['sort-name'] + discMD.sortName = credit.getSortName() # FIXME: is format str ? if not 'date' in release: log.warning('mbngs', 'Release %r does not have date', release) diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index a06b958..0350429 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -62,8 +62,8 @@ class MetadataTestCase(unittest.TestCase): metadata = mbngs._getMetadata({}, response['release'], discid) self.assertEquals(metadata.artist, u'Isobel Campbell & Mark Lanegan') - # FIXME: this should include Mark - self.assertEquals(metadata.sortName, u'Campbell, Isobel') + self.assertEquals(metadata.sortName, + u'Campbell, Isobel & Lanegan, Mark') self.assertEquals(metadata.release, u'2006-01-30') # FIXME: this is only Isobel self.assertEquals(metadata.mbidArtist, From ead648feef34979272931b737cc9881911fd9c5b Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:09:56 +0200 Subject: [PATCH 068/132] add getIds and use it to set correct ids --- morituri/common/mbngs.py | 15 +++++++++++---- morituri/test/test_common_mbngs.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index 1fca73e..6f8b460 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -114,14 +114,17 @@ class _Credit(list): or track. """ - def joiner(self, attributeGetter): + def joiner(self, attributeGetter, joinString=None): res = [] for item in self: if isinstance(item, dict): res.append(attributeGetter(item)) else: - res.append(item) + if not joinString: + res.append(item) + else: + res.append(joinString) return "".join(res) @@ -132,6 +135,10 @@ class _Credit(list): def getName(self): return self.joiner(lambda i: i.get('artist').get('name', None)) + def getIds(self): + return self.joiner(lambda i: i.get('artist').get('id', None), + joinString=";") + def _getMetadata(releaseShort, release, discid): """ @@ -165,7 +172,7 @@ def _getMetadata(releaseShort, release, discid): albumArtistName = credit.getName() - # FIXME: is there a better way to check for VA + # FIXME: is there a better way to check for VA ? discMD.various = False if discArtist['id'] == VA_ID: discMD.various = True @@ -180,7 +187,7 @@ def _getMetadata(releaseShort, release, discid): discMD.release = release['date'] discMD.mbid = release['id'] - discMD.mbidArtist = discArtist['id'] + discMD.mbidArtist = credit.getIds() discMD.url = 'http://musicbrainz.org/release/' + release['id'] discMD.barcode = release.get('barcode', None) diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index 0350429..89b2d2e 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -65,9 +65,9 @@ class MetadataTestCase(unittest.TestCase): self.assertEquals(metadata.sortName, u'Campbell, Isobel & Lanegan, Mark') self.assertEquals(metadata.release, u'2006-01-30') - # FIXME: this is only Isobel self.assertEquals(metadata.mbidArtist, - u'd51f3a15-12a2-41a0-acfa-33b5eae71164') + u'd51f3a15-12a2-41a0-acfa-33b5eae71164;' + 'a9126556-f555-4920-9617-6e013f8228a7') self.assertEquals(len(metadata.tracks), 12) From e5b2627db2703226b3dda308433569eb98bdefeb Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:16:15 +0200 Subject: [PATCH 069/132] add more FIXME --- morituri/test/test_common_mbngs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index 89b2d2e..4240eb0 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -49,6 +49,9 @@ class MetadataTestCase(unittest.TestCase): # of track artist id's self.assertEquals(track16.mbidArtist, u'89ad4ac3-39f7-470e-963a-56509c546377') + # FIXME: should not be various artists + self.assertEquals(track16.sortName, + u'Various Artists') def testBalladOfTheBrokenSeas(self): # various artists disc @@ -74,6 +77,11 @@ class MetadataTestCase(unittest.TestCase): track12 = metadata.tracks[11] self.assertEquals(track12.artist, u'Isobel Campbell & Mark Lanegan') + # FIXME: should include Mark Lanegan + self.assertEquals(track12.sortName, + u'Campbell, Isobel' + # ' & Lanegan, Mark' + ) # FIXME: this is only Isobel self.assertEquals(track12.mbidArtist, u'd51f3a15-12a2-41a0-acfa-33b5eae71164') From e8d06ba3f1b62552684c5665cfb8f6dcd720baaa Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:17:32 +0200 Subject: [PATCH 070/132] distinguish between disc and track credit --- morituri/common/mbngs.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index 6f8b460..a31f2f2 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -159,18 +159,18 @@ def _getMetadata(releaseShort, release, discid): discMD = DiscMetadata() discMD.releaseType = releaseShort.get('release-group', {}).get('type') - credit = _Credit(release['artist-credit']) + discCredit = _Credit(release['artist-credit']) # example: # [{'artist': # {'sort-name': 'Pixies', # 'id': 'b6b2bb8d-54a9-491f-9607-7b546023b433', 'name': 'Pixies'}}] - discArtist = credit[0]['artist'] + discArtist = discCredit[0]['artist'] - if len(credit) > 1: - log.debug('mbngs', 'artist-credit more than 1: %r', credit) + if len(discCredit) > 1: + log.debug('mbngs', 'artist-credit more than 1: %r', discCredit) - albumArtistName = credit.getName() + albumArtistName = discCredit.getName() # FIXME: is there a better way to check for VA ? discMD.various = False @@ -179,7 +179,7 @@ def _getMetadata(releaseShort, release, discid): # getUniqueName gets disambiguating names like Muse (UK rock band) discMD.artist = albumArtistName - discMD.sortName = credit.getSortName() + discMD.sortName = discCredit.getSortName() # FIXME: is format str ? if not 'date' in release: log.warning('mbngs', 'Release %r does not have date', release) @@ -187,7 +187,7 @@ def _getMetadata(releaseShort, release, discid): discMD.release = release['date'] discMD.mbid = release['id'] - discMD.mbidArtist = credit.getIds() + discMD.mbidArtist = discCredit.getIds() discMD.url = 'http://musicbrainz.org/release/' + release['id'] discMD.barcode = release.get('barcode', None) @@ -214,12 +214,12 @@ def _getMetadata(releaseShort, release, discid): discMD.title = title for t in medium['track-list']: track = TrackMetadata() - credit = _Credit(t['recording']['artist-credit']) - if len(credit) > 1: + trackCredit = _Credit(t['recording']['artist-credit']) + if len(trackCredit) > 1: log.debug('mbngs', - 'artist-credit more than 1: %r', credit) + 'artist-credit more than 1: %r', trackCredit) - trackArtistName = credit.getName() + trackArtistName = trackCredit.getName() if not discArtist: track.artist = discMD.artist From 7c95e8e0175ea215c3b569ce5f0643a097f40417 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:23:34 +0200 Subject: [PATCH 071/132] Set proper track ids. Fixes #34 --- morituri/common/mbngs.py | 6 ++++-- morituri/test/test_common_mbngs.py | 16 +++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index a31f2f2..88a1ae6 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -165,6 +165,8 @@ def _getMetadata(releaseShort, release, discid): # {'sort-name': 'Pixies', # 'id': 'b6b2bb8d-54a9-491f-9607-7b546023b433', 'name': 'Pixies'}}] + # FIXME: we should only use discArtist for determining VA; remove other + # uses discArtist = discCredit[0]['artist'] if len(discCredit) > 1: @@ -228,8 +230,8 @@ def _getMetadata(releaseShort, release, discid): else: # various artists discs can have tracks with no artist track.artist = trackArtistName - track.sortName = discArtist['sort-name'] - track.mbidArtist = discArtist['id'] + track.sortName = trackCredit.getSortName() + track.mbidArtist = trackCredit.getIds() track.title = t['recording']['title'] track.mbid = t['recording']['id'] diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index 4240eb0..830f7bd 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -45,13 +45,12 @@ class MetadataTestCase(unittest.TestCase): track16 = metadata.tracks[15] self.assertEquals(track16.artist, 'Tom Jones & Stereophonics') - # FIXME: this is the disc artist id, and it should be the combo - # of track artist id's self.assertEquals(track16.mbidArtist, - u'89ad4ac3-39f7-470e-963a-56509c546377') - # FIXME: should not be various artists + u'57c6f649-6cde-48a7-8114-2a200247601a' + ';0bfba3d3-6a04-4779-bb0a-df07df5b0558' + ) self.assertEquals(track16.sortName, - u'Various Artists') + u'Jones, Tom & Stereophonics') def testBalladOfTheBrokenSeas(self): # various artists disc @@ -77,11 +76,10 @@ class MetadataTestCase(unittest.TestCase): track12 = metadata.tracks[11] self.assertEquals(track12.artist, u'Isobel Campbell & Mark Lanegan') - # FIXME: should include Mark Lanegan self.assertEquals(track12.sortName, u'Campbell, Isobel' - # ' & Lanegan, Mark' + ' & Lanegan, Mark' ) - # FIXME: this is only Isobel self.assertEquals(track12.mbidArtist, - u'd51f3a15-12a2-41a0-acfa-33b5eae71164') + u'd51f3a15-12a2-41a0-acfa-33b5eae71164;' + 'a9126556-f555-4920-9617-6e013f8228a7') From 6eda5d7dc51b2c5083db2e86ec45597ce50bebb5 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:25:28 +0200 Subject: [PATCH 072/132] remove one if path --- morituri/common/mbngs.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index 88a1ae6..f84f101 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -221,17 +221,11 @@ def _getMetadata(releaseShort, release, discid): log.debug('mbngs', 'artist-credit more than 1: %r', trackCredit) - trackArtistName = trackCredit.getName() - - if not discArtist: - track.artist = discMD.artist - track.sortName = discMD.sortName - track.mbidArtist = discMD.mbidArtist - else: - # various artists discs can have tracks with no artist - track.artist = trackArtistName - track.sortName = trackCredit.getSortName() - track.mbidArtist = trackCredit.getIds() + # FIXME: leftover comment, need an example + # various artists discs can have tracks with no artist + track.artist = trackCredit.getName() + track.sortName = trackCredit.getSortName() + track.mbidArtist = trackCredit.getIds() track.title = t['recording']['title'] track.mbid = t['recording']['id'] From debd48d3e6cf8f5f0d6b42771c3ed5296092ef23 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:37:32 +0200 Subject: [PATCH 073/132] Add testcase to show that previous commit fixes #19. --- morituri/common/mbngs.py | 21 +++++------ morituri/test/Makefile.am | 3 +- ....61c6fd9b-18f8-4a45-963a-ba3c5d990cae.json | 1 + morituri/test/test_common_mbngs.py | 35 ++++++++++++++++++- 4 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 morituri/test/morituri.release.61c6fd9b-18f8-4a45-963a-ba3c5d990cae.json diff --git a/morituri/common/mbngs.py b/morituri/common/mbngs.py index f84f101..31a8883 100644 --- a/morituri/common/mbngs.py +++ b/morituri/common/mbngs.py @@ -106,6 +106,10 @@ def _record(record, which, name, what): # 'sort-name': 'Blackie and the Oohoos', # 'id': '028a9dc7-f5ef-43c2-866b-08d69ffff363', # 'name': 'Blackie & the Oohoos'}}] +# or +# [{'artist': +# {'sort-name': 'Pixies', +# 'id': 'b6b2bb8d-54a9-491f-9607-7b546023b433', 'name': 'Pixies'}}] class _Credit(list): @@ -160,25 +164,18 @@ def _getMetadata(releaseShort, release, discid): discMD.releaseType = releaseShort.get('release-group', {}).get('type') discCredit = _Credit(release['artist-credit']) - # example: - # [{'artist': - # {'sort-name': 'Pixies', - # 'id': 'b6b2bb8d-54a9-491f-9607-7b546023b433', 'name': 'Pixies'}}] - # FIXME: we should only use discArtist for determining VA; remove other - # uses - discArtist = discCredit[0]['artist'] + # FIXME: is there a better way to check for VA ? + discMD.various = False + if discCredit[0]['artist']['id'] == VA_ID: + discMD.various = True + if len(discCredit) > 1: log.debug('mbngs', 'artist-credit more than 1: %r', discCredit) albumArtistName = discCredit.getName() - # FIXME: is there a better way to check for VA ? - discMD.various = False - if discArtist['id'] == VA_ID: - discMD.various = True - # getUniqueName gets disambiguating names like Muse (UK rock band) discMD.artist = albumArtistName discMD.sortName = discCredit.getSortName() diff --git a/morituri/test/Makefile.am b/morituri/test/Makefile.am index 1b5b8d4..22bac66 100644 --- a/morituri/test/Makefile.am +++ b/morituri/test/Makefile.am @@ -37,8 +37,9 @@ EXTRA_DIST = \ release.08397059-86c1-463b-8ed0-cd596dbd174f.xml \ release.93a6268c-ddf1-4898-bf93-fb862b1c5c5e.xml \ release.c7d919f4-3ea0-4c4b-a230-b3605f069440.xml \ - morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json \ morituri.release.3451f29c-9bb8-4cc5-bfcc-bd50104b94f8.json \ + morituri.release.a76714e0-32b1-4ed4-b28e-f86d99642193.json \ + morituri.release.61c6fd9b-18f8-4a45-963a-ba3c5d990cae.json \ morituri.release.e32ae79a-336e-4d33-945c-8c5e8206dbd3.json \ kanye.cue \ kings-separate.cue \ diff --git a/morituri/test/morituri.release.61c6fd9b-18f8-4a45-963a-ba3c5d990cae.json b/morituri/test/morituri.release.61c6fd9b-18f8-4a45-963a-ba3c5d990cae.json new file mode 100644 index 0000000..de8f5d8 --- /dev/null +++ b/morituri/test/morituri.release.61c6fd9b-18f8-4a45-963a-ba3c5d990cae.json @@ -0,0 +1 @@ +{"release": {"status": "Official", "asin": "B008R78K1Y", "label-info-list": [{"label": {"sort-name": "Brownswood Recordings", "id": "6483a614-d00f-42b0-af39-a602b3ce5daa", "name": "Brownswood Recordings"}, "catalog-number": "BWOOD090CD"}], "title": "Mala in Cuba", "country": "GB", "barcode": "5060180321505", "artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "medium-list": [{"disc-list": [{"id": "u0aKVpO.59JBy6eQRX2vYcoqQZ0-", "sectors": "257868"}], "position": "1", "track-list": [{"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "155000", "artist-credit-phrase": "Mala", "id": "3fa9c442-6ae7-4242-ae3b-0150a3002da4", "title": "Introduction"}, "position": "1"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "195626", "artist-credit-phrase": "Mala", "id": "983ad5e0-c52e-459d-8828-85718ceff2cc", "title": "Mulata"}, "position": "2"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "242826", "artist-credit-phrase": "Mala", "id": "6855abf0-32a3-4fe2-a3fb-858f3157d42b", "title": "Tribal"}, "position": "3"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "263760", "artist-credit-phrase": "Mala", "id": "2f938885-94ad-4b11-b251-f18c3a2a5fa9", "title": "Changuito"}, "position": "4"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "274520", "artist-credit-phrase": "Mala", "id": "a5ecfa15-06d0-44cf-a28e-c748e8270488", "title": "Revolution"}, "position": "5"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}, " feat. ", {"artist": {"sort-name": "Dreiser", "id": "ec07a209-55ff-4084-bc41-9d4d1764e075", "name": "Dreiser"}}, " & ", {"artist": {"sort-name": "Sexto Sentido", "id": "f626b92e-07b1-4a19-ad13-c09d690db66c", "name": "Sexto Sentido"}}], "length": "227800", "artist-credit-phrase": "Mala feat. Dreiser & Sexto Sentido", "id": "cfb3ddaf-584c-4c86-b58c-752c63977bb8", "title": "Como como"}, "position": "6"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "276693", "artist-credit-phrase": "Mala", "id": "90da8ada-21e2-4e7b-ab46-ff04004a3d84", "title": "Cuba Electronic"}, "position": "7"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "267973", "artist-credit-phrase": "Mala", "id": "2bf67b46-30f5-4746-ab91-4c9675221a21", "title": "The Tunnel"}, "position": "8"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "246000", "artist-credit-phrase": "Mala", "id": "0cd61fa9-a97a-41e3-b3c3-db36f633b611", "title": "Ghost"}, "position": "9"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "250000", "artist-credit-phrase": "Mala", "id": "136989e9-f24f-4872-9026-1487869cc8de", "title": "Curfew"}, "position": "10"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "174000", "artist-credit-phrase": "Mala", "id": "26b6fd89-7021-4239-b6a7-76eca8c0515a", "title": "The Tourist"}, "position": "11"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "270733", "artist-credit-phrase": "Mala", "id": "62f7a892-f63b-4a2b-866f-db2a36533f8c", "title": "Change"}, "position": "12"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}], "length": "251853", "artist-credit-phrase": "Mala", "id": "4395c91a-d5e9-4fe4-92d2-deee3e0ebb5a", "title": "Calle F"}, "position": "13"}, {"recording": {"artist-credit": [{"artist": {"sort-name": "Mala", "id": "09f221eb-c97e-4da5-ac22-d7ab7c555bbb", "name": "Mala"}}, " feat. ", {"artist": {"sort-name": "Suarez, Danay", "id": "82f04998-7da8-4259-aa7f-d623e6ea2b91", "name": "Danay Suarez"}}], "length": "338000", "artist-credit-phrase": "Mala feat. Danay Suarez", "id": "e47a4fd9-8359-4a33-add8-e8c690e59055", "title": "Noche sue\u00f1os"}, "position": "14"}], "format": "CD"}], "text-representation": {"language": "eng", "script": "Latn"}, "date": "2012-09-17", "artist-credit-phrase": "Mala", "quality": "normal", "id": "61c6fd9b-18f8-4a45-963a-ba3c5d990cae"}} \ No newline at end of file diff --git a/morituri/test/test_common_mbngs.py b/morituri/test/test_common_mbngs.py index 830f7bd..6777bcc 100644 --- a/morituri/test/test_common_mbngs.py +++ b/morituri/test/test_common_mbngs.py @@ -48,7 +48,7 @@ class MetadataTestCase(unittest.TestCase): self.assertEquals(track16.mbidArtist, u'57c6f649-6cde-48a7-8114-2a200247601a' ';0bfba3d3-6a04-4779-bb0a-df07df5b0558' - ) + ) self.assertEquals(track16.sortName, u'Jones, Tom & Stereophonics') @@ -83,3 +83,36 @@ class MetadataTestCase(unittest.TestCase): self.assertEquals(track12.mbidArtist, u'd51f3a15-12a2-41a0-acfa-33b5eae71164;' 'a9126556-f555-4920-9617-6e013f8228a7') + + def testMalaInCuba(self): + # single artist disc, but with multiple artists tracks + # see https://github.com/thomasvs/morituri/issues/19 + path = os.path.join(os.path.dirname(__file__), + 'morituri.release.61c6fd9b-18f8-4a45-963a-ba3c5d990cae.json') + handle = open(path, "rb") + response = json.loads(handle.read()) + handle.close() + discid = "u0aKVpO.59JBy6eQRX2vYcoqQZ0-" + + metadata = mbngs._getMetadata({}, response['release'], discid) + + self.assertEquals(metadata.artist, u'Mala') + self.assertEquals(metadata.sortName, u'Mala') + self.assertEquals(metadata.release, u'2012-09-17') + self.assertEquals(metadata.mbidArtist, + u'09f221eb-c97e-4da5-ac22-d7ab7c555bbb') + + self.assertEquals(len(metadata.tracks), 14) + + track6 = metadata.tracks[5] + + self.assertEquals(track6.artist, u'Mala feat. Dreiser & Sexto Sentido') + self.assertEquals(track6.sortName, + u'Mala feat. Dreiser & Sexto Sentido') + self.assertEquals(track6.mbidArtist, + u'09f221eb-c97e-4da5-ac22-d7ab7c555bbb' + ';ec07a209-55ff-4084-bc41-9d4d1764e075' + ';f626b92e-07b1-4a19-ad13-c09d690db66c' + ) + + From f7f2340d74cbd5e38e694d8d845a361b28e1fd2c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 13:52:40 +0200 Subject: [PATCH 074/132] Releasing 0.2.2 --- NEWS | 13 +++++++++++-- RELEASE | 53 +++++++++------------------------------------------ configure.ac | 2 +- doc/release | 8 ++++++++ morituri.doap | 16 ++++++++++++++++ 5 files changed, 45 insertions(+), 47 deletions(-) diff --git a/NEWS b/NEWS index 0918e17..a3a6ce8 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,15 @@ -This is morituri 0.2.1, "married" +This is morituri 0.2.2, "my bad" -Coverage in 0.2.1: 67 % (1957 / 2889), 101 python tests +Coverage in 0.2.2: 67 % (1972 / 2904), 109 python tests + +Bugs fixed in 0.2.2: + +in github: +- 38: No matching offset found +- 35: 'rip cd info' should not eject the disc +- 34: Use track instead of the release artist MBID for the 'Musicbrainz artist ID' +- 33: "rip offset find" fails to initialise program.Program +- 19: Set album artist tag Features added in 0.2.1: diff --git a/RELEASE b/RELEASE index 88824f2..2351dcb 100644 --- a/RELEASE +++ b/RELEASE @@ -1,56 +1,21 @@ morituri is a CD ripper aiming for accuracy over speed. Its features are modeled to compare with Exact Audio Copy on Windows. -This is morituri 0.2.1 "married". +This is morituri 0.2.2, "my bad" -Coverage in 0.2.1: 67 % (1957 / 2889), 101 python tests +Coverage in 0.2.2: 67 % (1972 / 2904), 109 python tests -Features added in 0.2.1: - -- added "%X" template variable for uppercase filename extension -- added rip cd info -- added storing catalog number and barcode -- disambiguate releases with same name but different catno/barcode -- use all but last track to find offset -- add support to filter path names for better file system support -- add config options for path filtering -- fixes for older pyxdg and some versions of pycdio - -Bugs fixed in 0.2.1: - -in trac: -- 44: Optionally strip special characters from file names -- 121: ImportError: No module named CDDB -- 126: pycdio is no more optional : pkg_resources.DistributionNotFound: pycdio -- 135: rip drive analyze report "Cannot analyze the drive. Is there a CD in it?" when not able to defeat audio cache -- 137: pycdio returns an error when analyzing drive -- 138: Error when trying to rip with pycdio .19 -- 124: Checking of runtime dependencies +Bugs fixed in 0.2.2: in github: -- 31: Cryptic error message if xdg module is too old -- 30: AttributeError: Values instance has no attribute 'unknown' -- 26: Convert values returned from pycdio to str (workaround for upstream bug) -- 24: Filenames from musicbrainz may contain invalid characters for windows filesystems -- 23: Convert drive path from unicode to str when calling cdio.Device (pycdio 0.19 / Arch Linux) -- 22: Compare AccurateRip to num tracks -1, as last track not being checked -- 21: break up logger line -- 18: Crash if no path specified for '-O' option -- 17: Use XDG cache directory -- 16: Work with older versions of python-xdg -- 14: Use with statement to open files -- 13: Use os.path.join instead of hardcoded paths. -- 11: Ignore bash-compgen, to clean up git-status. -- 9: Ask which release to use if DiscID returns several matches -- 8: abort if invalid logger specified -- 7: Warn if no offset specified and no stored offset found/pycdio not available -- 6: Add "%X" template variable for uppercase filename extension. -- 3: (Optional) dependency on cddb should be documented -- 2: No module named moap.util -- dependency shoud be documented -- 1: No module named log -- use of submodules should be documented +- 38: No matching offset found +- 35: 'rip cd info' should not eject the disc +- 34: Use track instead of the release artist MBID for the 'Musicbrainz artist ID' +- 33: "rip offset find" fails to initialise program.Program +- 19: Set album artist tag -morituri 0.2.1 is brought to you by: +morituri 0.2.2 is brought to you by: Thomas Vander Stichele Velo Superman diff --git a/configure.ac b/configure.ac index f2e6445..7148eee 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl initialize autoconf dnl when going to/from release please remove/add the nano (fourth number) dnl releases only do Wall, trunk and prerelease does Werror too -AC_INIT(morituri, 0.2.1.1, +AC_INIT(morituri, 0.2.2, http://thomas.apestaart.org/morituri/trac/newticket, morituri) diff --git a/doc/release b/doc/release index f0fd883..ddfbf9e 100644 --- a/doc/release +++ b/doc/release @@ -22,6 +22,7 @@ pre-release checklist correct: https://thomas.apestaart.org/morituri/trac/query?order=priority&col=id&col=summary&col=status&col=type&col=priority&col=milestone&col=component&milestone=0.1.1 - remilestone still open tickets to next release +- FIXME: same on github release ------- @@ -38,11 +39,18 @@ release - Add list of new features to NEWS - Update bugs fixed in NEWS: moap doap bug query -f "- %(id)3s: %(summary)s" "milestone=$VERSION" + FIXME: same on github - Update README and home page with relevant new features, as well as version - Update RELEASE, copying sections from NEWS, and adding contributors with moap cl cont + or from git: + git log --format='%aN' | sort -u - Run moap cl prep and add comment about release - Update ChangeLog; add === release x.y.z === line +- commit locally + git commit -a -m "Releasing $VERSION" +- tag the release: + git tag -a v$VERSION -m "Releasing $VERSION" - make distcheck - make release - build rpm using rpmbuild and mach diff --git a/morituri.doap b/morituri.doap index 93d9525..b58e367 100644 --- a/morituri.doap +++ b/morituri.doap @@ -40,6 +40,22 @@ Morituri is a CD ripper aiming for maximum quality. + + + 0.2.2 + master + my bad + 2013-07-30 + + + +- fixed rip offset find +- set album and track artist id's properly +- rip cd info no longer ejects + + + + 0.2.1 From 7079a6b57f1a4dd2cdbcaae9952ab81d5c71be05 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 19:25:59 +0200 Subject: [PATCH 075/132] back to development --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 7148eee..efe8278 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl initialize autoconf dnl when going to/from release please remove/add the nano (fourth number) dnl releases only do Wall, trunk and prerelease does Werror too -AC_INIT(morituri, 0.2.2, +AC_INIT(morituri, 0.2.2.1, http://thomas.apestaart.org/morituri/trac/newticket, morituri) From c8b33c3b5b2d6fcdd5d0e67c82e55d1c2277813f Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 19:26:04 +0200 Subject: [PATCH 076/132] update release guide --- doc/release | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/release b/doc/release index ddfbf9e..d95d6ae 100644 --- a/doc/release +++ b/doc/release @@ -51,6 +51,9 @@ release git commit -a -m "Releasing $VERSION" - tag the release: git tag -a v$VERSION -m "Releasing $VERSION" +- make sure we build installed.py with the correct tag + autoregen.sh + cat morituri/configure/installed.py - make distcheck - make release - build rpm using rpmbuild and mach @@ -64,9 +67,8 @@ release cp morituri-$VERSION.tar* /home/thomas/www/thomas.apestaart.org/data/download/morituri cp /var/tmp/mach/fedora-*/morituri-$VERSION-*/*.rpm /home/thomas/www/thomas.apestaart.org/data/download/morituri tao-put -- commit to master -- create release tag: - git tag -a v$VERSION +- push to master + git push git push origin v$VERSION - announce to freshmeat: moap doap -v $VERSION freshmeat From 5ef0fa3fa9e9d80849b44fa2273ea1c07ed158fd Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 19:28:23 +0200 Subject: [PATCH 077/132] add flattr link --- README | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README b/README index 191d53d..7d726ca 100644 --- a/README +++ b/README @@ -148,6 +148,12 @@ There is a "main" section and zero or more sections starting with "drive:" - defeats_cache: whether this drive can defeat the audio cache - read_offset: the read offset of the drive +CONTRIBUTING +------------ +- Please send pull requests through github. +- You can always flattr morituri to donate: + https://flattr.com/submit/auto?%20%20user_id=thomasvs&url=https://thomas.apestaart.org/morituri/trac/&%20%20title=morituri&%20%20description=morituri&%20%20language=en_GB&tags=flattr,morituri,software&category=software + rip command tree ---------------- From b7880f6206f468923c20ad664079e8b787e1b776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomo=20Lempi=C3=A4inen?= Date: Tue, 30 Jul 2013 22:01:53 +0300 Subject: [PATCH 078/132] Run $PYTHON, not python --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index efe8278..3f38cc5 100644 --- a/configure.ac +++ b/configure.ac @@ -39,7 +39,7 @@ AS_AC_EXPAND(PLUGINSDIR, "\${libdir}/morituri/plugins") AC_MSG_NOTICE(Setting plugins directory to $PLUGINSDIR) dnl get git revision for installed.py.in -AC_SUBST(REVISION, `python -c 'from morituri.configure import configure; print configure.revision'`) +AC_SUBST(REVISION, `$PYTHON -c 'from morituri.configure import configure; print configure.revision'`) AC_MSG_NOTICE(Setting revision to $REVISION) dnl check for epydoc From 4dc27e6e0a8d733683f34842dd4f0fe54e37295c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 30 Jul 2013 23:17:24 +0200 Subject: [PATCH 079/132] suggestions from hydrogen audio --- RELEASE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE b/RELEASE index 2351dcb..5edf18f 100644 --- a/RELEASE +++ b/RELEASE @@ -1,5 +1,7 @@ morituri is a CD ripper aiming for accuracy over speed. +morituri runs on Linux and possibly other Unix-based systems. Its features are modeled to compare with Exact Audio Copy on Windows. +For more information, see http://thomas.apestaart.org/morituri/trac/ This is morituri 0.2.2, "my bad" From df0daefa27f6911167c73424ccac1c3d9480abf2 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Wed, 31 Jul 2013 12:18:41 +0200 Subject: [PATCH 080/132] use a built REVISION file as a backup revision source --- .gitignore | 1 + Makefile.am | 5 ++++- morituri/common/common.py | 27 +++++++++++++++++++++++---- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index ff01510..0fbf43a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ install-sh missing morituri.spec py-compile +REVISION diff --git a/Makefile.am b/Makefile.am index dc8e9f7..8d8d232 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,7 +5,7 @@ ACLOCAL_AMFLAGS = -I m4 SUBDIRS = morituri bin etc doc m4 misc -EXTRA_DIST = morituri.spec morituri.doap RELEASE README HACKING +EXTRA_DIST = morituri.spec morituri.doap RELEASE README HACKING REVISION SOURCES = $(top_srcdir)/morituri/*.py $(top_srcdir)/morituri/*/*.py @@ -38,6 +38,9 @@ PYCHECKER_BLACKLIST = \ release: dist make $(PACKAGE)-$(VERSION).tar.bz2.md5 +REVISION: $(top_srcdir)/.git + $(PYTHON) -c 'from morituri.configure import configure; print configure.revision' > REVISION + # generate md5 sum files %.md5: % md5sum $< > $@ diff --git a/morituri/common/common.py b/morituri/common/common.py index aa964e9..1a30eaf 100644 --- a/morituri/common/common.py +++ b/morituri/common/common.py @@ -339,10 +339,29 @@ def getRevision(): Get a revision tag for the current git source tree. Appends -modified in case there are local modifications. + + If this is not a git tree, return the top-level REVISION contents instead. + + Finally, return unknown. """ - describe = commands.getoutput('git describe') + topsrcdir = os.path.join(os.path.dirname(__file__), '..', '..') - if commands.getoutput('git diff-index --name-only HEAD --'): - describe += '-modified' + # only use git if our src directory looks like a git checkout + # if you run git regardless, it recurses up until it finds a .git, + # which may be higher than your current source tree + if os.path.exists(os.path.join(topsrcdir, '.git')): - return describe + status, describe = commands.getstatusoutput('git describe') + if status == 0: + if commands.getoutput('git diff-index --name-only HEAD --'): + describe += '-modified' + + return describe + + # check for a top-level REIVISION file + path = os.path.join(topsrcdir, 'REVISION') + if os.path.exists(path): + revision = open(path).read().strip() + return revision + + return '(unknown)' From 120d1b239f7204bd8305a284e0fbcb11f83fe4b4 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 4 Aug 2013 18:11:13 +0200 Subject: [PATCH 081/132] work without network --- morituri/common/program.py | 20 ++++++++++++++++---- morituri/rip/cd.py | 20 +++++++++++++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/morituri/common/program.py b/morituri/common/program.py index 4bdced7..a65b9c0 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -33,6 +33,7 @@ from morituri.program import cdrdao, cdparanoia from morituri.image import image from morituri.extern.task import task, gstreamer +from morituri.extern.musicbrainzngs import musicbrainz # FIXME: should Program have a runner ? @@ -297,10 +298,18 @@ class Program(log.Loggable): """ # FIXME: convert to nonblocking? import CDDB - code, md = CDDB.query(cddbdiscid) - self.debug('CDDB query result: %r, %r', code, md) - if code == 200: - return md['title'] + try: + code, md = CDDB.query(cddbdiscid) + self.debug('CDDB query result: %r, %r', code, md) + if code == 200: + return md['title'] + + except IOError, e: + # FIXME: for some reason errno is a str ? + if e.errno == 'socket error': + self._stdout.write("Warning: network error: %r\n" % (e, )) + else: + raise return None @@ -325,6 +334,9 @@ class Program(log.Loggable): record=self._record) except mbngs.NotFoundException, e: break + except musicbrainz.NetworkError, e: + self._stdout.write("Warning: network error: %r\n" % (e, )) + break except mbngs.MusicBrainzException, e: self._stdout.write("Warning: %r\n" % (e, )) time.sleep(5) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index c2c1a94..1689ae0 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -23,6 +23,8 @@ import os import math import glob +import urllib2 +import socket import gobject gobject.threads_init() @@ -97,8 +99,9 @@ class _CD(logcommand.LogCommand): self.stdout.write('FreeDB identifies disc as %s\n' % cddbmd) # also used by rip cd info - if not getattr(self.options, 'unknown', False) and self.eject: - self.program.ejectDevice(self.device) + if not getattr(self.options, 'unknown', False): + if self.eject: + self.program.ejectDevice(self.device) return -1 # now, read the complete index table, which is slower @@ -478,7 +481,18 @@ Install pycdio and run 'rip offset find' to detect your drive's offset. self.stdout.write("AccurateRip URL %s\n" % url) accucache = accurip.AccuCache() - responses = accucache.retrieve(url) + try: + responses = accucache.retrieve(url) + except urllib2.URLError, e: + if isinstance(e.args[0], socket.gaierror): + if e.args[0].errno == -2: + self.stdout.write("Warning: network error: %r\n" % ( + e.args[0], )) + responses = None + else: + raise + else: + raise if not responses: self.stdout.write('Album not found in AccurateRip database\n') From 7f3a7a33d10da965f9b04c32fb9ce3e9802b4766 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 22 Aug 2013 15:38:05 +0200 Subject: [PATCH 082/132] README: slight markup modifications to show formatted on github start page --- README => README.md | 145 +++++++++++++++++++++++--------------------- 1 file changed, 76 insertions(+), 69 deletions(-) rename README => README.md (54%) diff --git a/README b/README.md similarity index 54% rename from README rename to README.md index 7d726ca..8fcf713 100644 --- a/README +++ b/README.md @@ -4,8 +4,8 @@ Its features are modeled to compare with Exact Audio Copy on Windows. RATIONALE --------- -For a more detailed rationale, see my wiki page 'The Art of the Rip' -at https://thomas.apestaart.org/thomas/trac/wiki/DAD/Rip +For a more detailed rationale, see my wiki page ['The Art of the Rip']( +https://thomas.apestaart.org/thomas/trac/wiki/DAD/Rip). FEATURES -------- @@ -38,43 +38,52 @@ If you are building from a source tarball or checkout, you can choose to use morituri installed or uninstalled. - getting: - - Change to a directory where you want to put the morituri source code - (For example, $HOME/dev/ext or $HOME/prefix/src) - - source: download tarball, unpack, and change to its directory - - checkout: - git clone git://github.com/thomasvs/morituri.git - cd morituri - git submodule init - git submodule update - ./autogen.sh + - Change to a directory where you want to put the morituri source code + (For example, `$HOME/dev/ext` or `$HOME/prefix/src`) + - source: download tarball, unpack, and change to its directory + - checkout: + + git clone git://github.com/thomasvs/morituri.git + cd morituri + git submodule init + git submodule update + ./autogen.sh - building: - ./configure - make + + ./configure + make - you can now choose to install it or run it uninstalled. - - installing: - make install - - running uninstalled: - ln -sf `pwd`/misc/morituri-uninstalled $HOME/bin/morituri-git - morituri-git - (this drops you in a shell where everything is set up to use morituri) + + - installing: + + make install + + - running uninstalled: + + ln -sf `pwd`/misc/morituri-uninstalled $HOME/bin/morituri-git + morituri-git # this drops you in a shell where everything is set up to use morituri RUNNING MORITURI ---------------- morituri currently only has a command-line interface called 'rip' rip is self-documenting. -rip -h gives you the basic instructions. +`rip -h` gives you the basic instructions. rip implements a tree of commands; for example, the top-level 'changelog' command has a number of sub-commands. Positioning of arguments is important; - rip cd -d (device) rip + + rip cd -d (device) rip + is correct, while - rip cd rip -d (device) -is not, because the -d argument applies to the rip command. + + rip cd rip -d (device) + +is not, because the `-d` argument applies to the rip command. Check the man page (rip(1)) for more information. @@ -85,9 +94,9 @@ RUNNING UNINSTALLED To make it easier for developers, you can run morituri straight from the source checkout: -./autogen.sh -make -misc/morituri-uninstalled + ./autogen.sh + make + misc/morituri-uninstalled GETTING STARTED --------------- @@ -96,22 +105,29 @@ The simplest way to get started making accurate rips is: - pick a relatively popular CD that has a good change of being in the AccurateRip database - find the drive's offset by running - rip offset find + + rip offset find + - wait for it to complete; this might take a while - optionally, confirm this offset with two more discs - analyze the drive's caching behaviour - rip drive analyze -- rip the disc by running - rip cd rip --offset (the number you got before) + + rip drive analyze + +- rip the disc by running one of + + rip cd rip # uses the offset from configuration file + rip cd rip --offset (the number you got before) # manually specified offset FILING BUGS ----------- -morituri's bug tracker is at https://thomas.apestaart.org/morituri/trac/ +morituri's bug tracker is at [https://thomas.apestaart.org/morituri/trac/]( +https://thomas.apestaart.org/morituri/trac/). When filing bugs, please run the failing command with the environment variable -RIP_DEBUG set; for example: +`RIP_DEBUG` set; for example: - RIP_DEBUG=5 rip offset find > morituri.log 2>&1 - gzip morituri.log + RIP_DEBUG=5 rip offset find > morituri.log 2>&1 + gzip morituri.log And attach the gzipped log file to your bug report. @@ -129,58 +145,49 @@ GOALS CONFIGURATION FILE ------------------ -The configuration file is stored according to -http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html +The configuration file is stored according to [XDG Base Directory Specification]( +http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html) when possible. -It lives in $XDG_CONFIG_HOME/morituri/morituri.conf +It lives in `$XDG_CONFIG_HOME/morituri/morituri.conf` The configuration file follows python's ConfigParser syntax. There is a "main" section and zero or more sections starting with "drive:" - main section: - - path_filter_fat: whether to filter path components for FAT file systems - - path_filter_special: whether to filter path components for special - characters + - `path_filter_fat`: whether to filter path components for FAT file systems + - `path_filter_special`: whether to filter path components for special + characters - drive section: All these values are probed by morituri and should not be edited by hand. - - defeats_cache: whether this drive can defeat the audio cache - - read_offset: the read offset of the drive + - `defeats_cache`: whether this drive can defeat the audio cache + - `read_offset`: the read offset of the drive CONTRIBUTING ------------ - Please send pull requests through github. -- You can always flattr morituri to donate: - https://flattr.com/submit/auto?%20%20user_id=thomasvs&url=https://thomas.apestaart.org/morituri/trac/&%20%20title=morituri&%20%20description=morituri&%20%20language=en_GB&tags=flattr,morituri,software&category=software +- You can always [flattr morituri to donate](https://flattr.com/submit/auto?%20%20user_id=thomasvs&url=https://thomas.apestaart.org/morituri/trac/&%20%20title=morituri&%20%20description=morituri&%20%20language=en_GB&tags=flattr,morituri,software&category=software) rip command tree ---------------- rip - accurip - show - show accuraterip data - offset - find - find drive's read offset using AccurateRip - verify - verify drive's read offset using AccurateRip - cd - rip - rip the cd - debug - encode - encode a file - htoa - find - rip - rip the htoa if it's there - image - verify - verify the cd image - encode - encode to a different codec - retag - retag the image with current MusicBrainz data + + * accurip + * show (show accuraterip data) + * offset + * find (find drive's read offset using AccurateRip) + * verify (verify drive's read offset using AccurateRip) + * cd + * rip (rip the cd) + * debug + * encode (encode a file) + * htoa + * find + * rip (rip the htoa if it's there) + * image + * verify (verify the cd image) + * encode (encode to a different codec) + * retag (retag the image with current MusicBrainz data) From 1e9886d42231188f3fb86b62ca3b2d87882c8751 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 00:33:57 +0200 Subject: [PATCH 083/132] handle the case where we couldn't even import deps. Fixes ticket 147 on trac. --- bin/rip.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/rip.in b/bin/rip.in index ac38958..6f7cccf 100755 --- a/bin/rip.in +++ b/bin/rip.in @@ -30,6 +30,8 @@ and assure it doesn't raise an exception. sys.exit(1) # now load the main function +h = None + try: from morituri.common import deps from morituri.extern.deps import deps as edeps @@ -38,6 +40,9 @@ try: from morituri.rip import main sys.exit(main.main(sys.argv[1:])) except ImportError, e: + if not h: + # we couldn't even import deps, so reraise + raise h.handleImportError(e) sys.exit(1) except edeps.DependencyError: From d4365184c8f38e49741a24464bd164bc6dc5fc31 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 01:09:53 +0200 Subject: [PATCH 084/132] add make rules to trigger git submodule init/sync/update Fixes trac bug #144 --- morituri/extern/Makefile.am | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/morituri/extern/Makefile.am b/morituri/extern/Makefile.am index c39716d..2424f60 100644 --- a/morituri/extern/Makefile.am +++ b/morituri/extern/Makefile.am @@ -44,4 +44,10 @@ musicbrainzngs_PYTHON = \ EXTRA_DIST = python-command/scripts/help2man -musicbrainzngs/musicbrainz.py: all +flog/log.py: git-submodule +musicbrainzngs/musicbrainz.py: git-submodule + +git-submodule: + cd $(top_srcdir) && git submodule init && git submodule sync && git submodule update + +all-local: flog/log.py musicbrainzngs/musicbrainz.py From 987bc91c2ffaaef4f3f35a7bfa8fa0179b7296d9 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 01:38:41 +0200 Subject: [PATCH 085/132] make sure distcheck works too --- morituri/extern/Makefile.am | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/morituri/extern/Makefile.am b/morituri/extern/Makefile.am index 2424f60..327739b 100644 --- a/morituri/extern/Makefile.am +++ b/morituri/extern/Makefile.am @@ -44,10 +44,15 @@ musicbrainzngs_PYTHON = \ EXTRA_DIST = python-command/scripts/help2man -flog/log.py: git-submodule -musicbrainzngs/musicbrainz.py: git-submodule +log: + make git-submodule +musicbrainzngs: + make git-submodule git-submodule: cd $(top_srcdir) && git submodule init && git submodule sync && git submodule update -all-local: flog/log.py musicbrainzngs/musicbrainz.py +.PHONY: git-submodule + + +all-local: log musicbrainzngs From 1a047a2ebe2fb01d733f30577ac12173efb164aa Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 01:51:44 +0200 Subject: [PATCH 086/132] hack to get proper revision before generating configure/*py --- morituri/configure/configure.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/morituri/configure/configure.py b/morituri/configure/configure.py index 6a38329..3d626f4 100644 --- a/morituri/configure/configure.py +++ b/morituri/configure/configure.py @@ -25,9 +25,15 @@ __thisdir = os.path.dirname(os.path.abspath(__file__)) if os.path.exists(os.path.join(__thisdir, 'uninstalled.py')): from morituri.configure import uninstalled config_dict = uninstalled.get() -else: +elif os.path.exists(os.path.join(__thisdir, 'installed.py')): from morituri.configure import installed config_dict = installed.get() +else: + # hack on fresh checkout, no make run yet, and configure needs revision + from morituri.common import common + config_dict = { + 'revision': common.getRevision(), + } for key, value in config_dict.items(): dictionary = locals() From f30b8b598f7b626ca11677cf8cb74afb47657e20 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 3 Mar 2013 19:39:35 +0100 Subject: [PATCH 087/132] add more asserts on the current code for strokes this shows that some of the logic may be wrong; track 1 index 1 should have relative 0 as it draws from the file, not silence --- morituri/test/test_image_toc.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 24e9345..2ffebec 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -335,8 +335,23 @@ class StrokesTestCase(common.TestCase): def testIndexes(self): t = self.toc.table.tracks[0] - self.assertEquals(t.getIndex(0).relative, 0) - self.assertEquals(t.getIndex(1).relative, 1) + i0 = t.getIndex(0) + self.assertEquals(i0.relative, 0) + self.assertEquals(i0.absolute, 0) + # FIXME: this is what we should have + # self.assertEquals(i0.counter, 0) + # self.assertEquals(i0.path, None) + # FIXME: this is what it is right now + self.assertEquals(i0.counter, 1) + self.assertEquals(i0.path, u'data.wav') + + i1 = t.getIndex(1) + # FIXME: relative should be 0, as it should point to the start + # of data.wav + self.assertEquals(i1.relative, 1) + self.assertEquals(i1.absolute, 1) + self.assertEquals(i1.counter, 1) + self.assertEquals(i1.path, u'data.wav') # Surfer Rosa has From a8fe47f1bd1eb6604c744a9fd5b29e6d6dac9347 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 3 Mar 2013 23:11:11 +0100 Subject: [PATCH 088/132] compare output .cue with EAC one we add The EAC one clearly marks the HTOA of one frame as pre-gap, while we don't yet do that. --- morituri/test/strokes-someday.eac.cue | 13 +++++++++++ morituri/test/test_image_toc.py | 31 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 morituri/test/strokes-someday.eac.cue diff --git a/morituri/test/strokes-someday.eac.cue b/morituri/test/strokes-someday.eac.cue new file mode 100644 index 0000000..0180794 --- /dev/null +++ b/morituri/test/strokes-someday.eac.cue @@ -0,0 +1,13 @@ +REM GENRE "Alternative Rock" +REM DATE 2001 +REM DISCID 0200BA01 +REM COMMENT "ExactAudioCopy v0.99pb4" +PERFORMER "The Strokes" +TITLE "Someday" +FILE "The Strokes - Someday\01 - The Strokes - Someday.wav" WAVE + TRACK 01 AUDIO + TITLE "Someday" + PERFORMER "The Strokes" + FLAGS DCP + PREGAP 00:00:01 + INDEX 01 00:00:00 diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 2ffebec..e41bc73 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -353,6 +353,37 @@ class StrokesTestCase(common.TestCase): self.assertEquals(i1.counter, 1) self.assertEquals(i1.path, u'data.wav') + cue = self._filterCue(self.toc.table.cue()) + ref = self._filterCue(open(os.path.join(os.path.dirname(__file__), + 'strokes-someday.eac.cue')).read()) + # FIXME: this diff should match + # common.diffStrings(cue, ref) + self.assertRaises(AssertionError, common.diffStrings, cue, ref) + + def _filterCue(self, output): + # helper to be able to compare our generated .cue with the + # EAC-extracted one + discard = [ 'TITLE', 'PERFORMER', 'FLAGS', 'REM' ] + lines = output.split('\n') + + res = [] + + for line in lines: + found = False + for needle in discard: + if line.find(needle) > -1: + found = True + + if line.find('FILE') > -1: + line = 'FILE "data.wav" WAVE' + + if not found: + res.append(line) + + return '\n'.join(res) + + + # Surfer Rosa has # track 00 consisting of 32 frames of SILENCE From 879dd7f946f78143195d7a24fe90d4a3d704cc11 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 3 Mar 2013 23:14:39 +0100 Subject: [PATCH 089/132] reset relative offset on SILENCE/ZERO --- morituri/image/toc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index b43ee41..7e19775 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -209,6 +209,7 @@ class TocFile(object, log.Loggable): if currentFile is not None: self.debug('SILENCE after FILE, increasing counter') counter += 1 + relativeOffset = 0 currentFile = None currentLength += common.msfToFrames(length) @@ -218,6 +219,7 @@ class TocFile(object, log.Loggable): if currentFile is not None: self.debug('ZERO after FILE, increasing counter') counter += 1 + relativeOffset = 0 currentFile = None length = m.group('length') currentLength += common.msfToFrames(length) From 1271ccea18c63088e79f32cd1f52b30105a675e5 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 3 Mar 2013 23:15:44 +0100 Subject: [PATCH 090/132] Set start and length properly on File --- morituri/image/toc.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 7e19775..4190324 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -239,7 +239,8 @@ class TocFile(object, log.Loggable): self.debug('track %d, switched to new FILE, ' 'increased counter to %d', trackNumber, counter) - currentFile = File(filePath, start, length) + currentFile = File(filePath, common.msfToFrames(start), + common.msfToFrames(length)) #absoluteOffset += common.msfToFrames(start) currentLength += common.msfToFrames(length) @@ -258,7 +259,7 @@ class TocFile(object, log.Loggable): 'increased counter to %d', trackNumber, counter) # FIXME: assume that a MODE2_FORM_MIX track always starts at 0 - currentFile = File(filePath, 0, length) + currentFile = File(filePath, 0, common.msfToFrames(length)) #absoluteOffset += common.msfToFrames(start) currentLength += common.msfToFrames(length) @@ -353,13 +354,16 @@ class File: def __init__(self, path, start, length): """ - @type path: unicode + @type path: C{unicode} + @type start: C{int} + @param start: starting point for the track in this file, in frames + @param length: length for the track in this file, in frames """ assert type(path) is unicode, "%r is not unicode" % path self.path = path - #self.start = start - #self.length = length + self.start = start + self.length = length def __repr__(self): return '' % (self.path, ) From 16bdaac89edd52401a77c61468ffb84628a90ca0 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 19 Aug 2013 00:21:27 +0200 Subject: [PATCH 091/132] add all EAC cue files for surfer rosa --- morituri/test/surferrosa.eac.corrected.cue | 136 ++++++++++++++++++ morituri/test/surferrosa.eac.currentgap.cue | 136 ++++++++++++++++++ morituri/test/surferrosa.eac.leftout.cue | 136 ++++++++++++++++++ morituri/test/surferrosa.eac.noncompliant.cue | 136 ++++++++++++++++++ morituri/test/surferrosa.eac.single.cue | 116 +++++++++++++++ 5 files changed, 660 insertions(+) create mode 100644 morituri/test/surferrosa.eac.corrected.cue create mode 100644 morituri/test/surferrosa.eac.currentgap.cue create mode 100644 morituri/test/surferrosa.eac.leftout.cue create mode 100644 morituri/test/surferrosa.eac.noncompliant.cue create mode 100644 morituri/test/surferrosa.eac.single.cue diff --git a/morituri/test/surferrosa.eac.corrected.cue b/morituri/test/surferrosa.eac.corrected.cue new file mode 100644 index 0000000..bf764b3 --- /dev/null +++ b/morituri/test/surferrosa.eac.corrected.cue @@ -0,0 +1,136 @@ +REM GENRE Alternative +REM DATE 1987 +REM DISCID 350CAA15 +REM COMMENT "ExactAudioCopy v0.99pb4" +CATALOG 0000000000000 +PERFORMER "Pixies" +TITLE "Surfer Rosa & Come on Pilgrim" +FILE "Pixies - Surfer Rosa & Come on Pilgrim\01 - Pixies - Bone Machine.wav" WAVE + TRACK 01 AUDIO + TITLE "Bone Machine" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 00 00:00:00 + INDEX 01 00:00:32 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\02 - Pixies - Break My Body.wav" WAVE + TRACK 02 AUDIO + TITLE "Break My Body" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\03 - Pixies - Something Against You.wav" WAVE + TRACK 03 AUDIO + TITLE "Something Against You" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 00 00:00:00 + INDEX 01 00:00:45 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\04 - Pixies - Broken Face.wav" WAVE + TRACK 04 AUDIO + TITLE "Broken Face" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\05 - Pixies - Gigantic.wav" WAVE + TRACK 05 AUDIO + TITLE "Gigantic" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\06 - Pixies - River Euphrates.wav" WAVE + TRACK 06 AUDIO + TITLE "River Euphrates" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\07 - Pixies - Where Is My Mind .wav" WAVE + TRACK 07 AUDIO + TITLE "Where Is My Mind?" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\08 - Pixies - Cactus.wav" WAVE + TRACK 08 AUDIO + TITLE "Cactus" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\09 - Pixies - Tony's Theme.wav" WAVE + TRACK 09 AUDIO + TITLE "Tony's Theme" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\10 - Pixies - Oh My Golly!.wav" WAVE + TRACK 10 AUDIO + TITLE "Oh My Golly!" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\11 - Pixies - Vamos.wav" WAVE + TRACK 11 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 + INDEX 02 00:44:70 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\12 - Pixies - I'm Amazed.wav" WAVE + TRACK 12 AUDIO + TITLE "I'm Amazed" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\13 - Pixies - Brick is Red.wav" WAVE + TRACK 13 AUDIO + TITLE "Brick is Red" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\14 - Pixies - Caribou.wav" WAVE + TRACK 14 AUDIO + TITLE "Caribou" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\15 - Pixies - Vamos.wav" WAVE + TRACK 15 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\16 - Pixies - Isla de Encanta.wav" WAVE + TRACK 16 AUDIO + TITLE "Isla de Encanta" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\17 - Pixies - Ed is Dead.wav" WAVE + TRACK 17 AUDIO + TITLE "Ed is Dead" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\18 - Pixies - The Holyday Song.wav" WAVE + TRACK 18 AUDIO + TITLE "The Holyday Song" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\19 - Pixies - Nimrod's Son.wav" WAVE + TRACK 19 AUDIO + TITLE "Nimrod's Son" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\20 - Pixies - I've Been Tired.wav" WAVE + TRACK 20 AUDIO + TITLE "I've Been Tired" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\21 - Pixies - Levitate Me.wav" WAVE + TRACK 21 AUDIO + TITLE "Levitate Me" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 diff --git a/morituri/test/surferrosa.eac.currentgap.cue b/morituri/test/surferrosa.eac.currentgap.cue new file mode 100644 index 0000000..9e7f189 --- /dev/null +++ b/morituri/test/surferrosa.eac.currentgap.cue @@ -0,0 +1,136 @@ +REM GENRE Alternative +REM DATE 1987 +REM DISCID 350CAA15 +REM COMMENT "ExactAudioCopy v0.99pb4" +CATALOG 0000000000000 +PERFORMER "Pixies" +TITLE "Surfer Rosa & Come on Pilgrim" +FILE "Pixies - Surfer Rosa & Come on Pilgrim\01 - Pixies - Bone Machine.wav" WAVE + TRACK 01 AUDIO + TITLE "Bone Machine" + PERFORMER "Pixies" + ISRC 000000000000 + PREGAP 00:00:32 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\02 - Pixies - Break My Body.wav" WAVE + TRACK 02 AUDIO + TITLE "Break My Body" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 + TRACK 03 AUDIO + TITLE "Something Against You" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 00 02:05:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\03 - Pixies - Something Against You.wav" WAVE + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\04 - Pixies - Broken Face.wav" WAVE + TRACK 04 AUDIO + TITLE "Broken Face" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\05 - Pixies - Gigantic.wav" WAVE + TRACK 05 AUDIO + TITLE "Gigantic" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\06 - Pixies - River Euphrates.wav" WAVE + TRACK 06 AUDIO + TITLE "River Euphrates" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\07 - Pixies - Where Is My Mind .wav" WAVE + TRACK 07 AUDIO + TITLE "Where Is My Mind?" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\08 - Pixies - Cactus.wav" WAVE + TRACK 08 AUDIO + TITLE "Cactus" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\09 - Pixies - Tony's Theme.wav" WAVE + TRACK 09 AUDIO + TITLE "Tony's Theme" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\10 - Pixies - Oh My Golly!.wav" WAVE + TRACK 10 AUDIO + TITLE "Oh My Golly!" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\11 - Pixies - Vamos.wav" WAVE + TRACK 11 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 + INDEX 02 00:44:70 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\12 - Pixies - I'm Amazed.wav" WAVE + TRACK 12 AUDIO + TITLE "I'm Amazed" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\13 - Pixies - Brick is Red.wav" WAVE + TRACK 13 AUDIO + TITLE "Brick is Red" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\14 - Pixies - Caribou.wav" WAVE + TRACK 14 AUDIO + TITLE "Caribou" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\15 - Pixies - Vamos.wav" WAVE + TRACK 15 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\16 - Pixies - Isla de Encanta.wav" WAVE + TRACK 16 AUDIO + TITLE "Isla de Encanta" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\17 - Pixies - Ed is Dead.wav" WAVE + TRACK 17 AUDIO + TITLE "Ed is Dead" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\18 - Pixies - The Holyday Song.wav" WAVE + TRACK 18 AUDIO + TITLE "The Holyday Song" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\19 - Pixies - Nimrod's Son.wav" WAVE + TRACK 19 AUDIO + TITLE "Nimrod's Son" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\20 - Pixies - I've Been Tired.wav" WAVE + TRACK 20 AUDIO + TITLE "I've Been Tired" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\21 - Pixies - Levitate Me.wav" WAVE + TRACK 21 AUDIO + TITLE "Levitate Me" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 diff --git a/morituri/test/surferrosa.eac.leftout.cue b/morituri/test/surferrosa.eac.leftout.cue new file mode 100644 index 0000000..b32adb4 --- /dev/null +++ b/morituri/test/surferrosa.eac.leftout.cue @@ -0,0 +1,136 @@ +REM GENRE Alternative +REM DATE 1987 +REM DISCID 350CAA15 +REM COMMENT "ExactAudioCopy v0.99pb4" +CATALOG 0000000000000 +PERFORMER "Pixies" +TITLE "Surfer Rosa & Come on Pilgrim" +FILE "Pixies - Surfer Rosa & Come on Pilgrim\01 - Pixies - Bone Machine.wav" WAVE + TRACK 01 AUDIO + TITLE "Bone Machine" + PERFORMER "Pixies" + ISRC 000000000000 + PREGAP 00:00:32 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\02 - Pixies - Break My Body.wav" WAVE + TRACK 02 AUDIO + TITLE "Break My Body" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\03 - Pixies - Something Against You.wav" WAVE + TRACK 03 AUDIO + TITLE "Something Against You" + PERFORMER "Pixies" + ISRC 000000000000 + PREGAP 00:00:45 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\04 - Pixies - Broken Face.wav" WAVE + TRACK 04 AUDIO + TITLE "Broken Face" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\05 - Pixies - Gigantic.wav" WAVE + TRACK 05 AUDIO + TITLE "Gigantic" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\06 - Pixies - River Euphrates.wav" WAVE + TRACK 06 AUDIO + TITLE "River Euphrates" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\07 - Pixies - Where Is My Mind .wav" WAVE + TRACK 07 AUDIO + TITLE "Where Is My Mind?" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\08 - Pixies - Cactus.wav" WAVE + TRACK 08 AUDIO + TITLE "Cactus" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\09 - Pixies - Tony's Theme.wav" WAVE + TRACK 09 AUDIO + TITLE "Tony's Theme" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\10 - Pixies - Oh My Golly!.wav" WAVE + TRACK 10 AUDIO + TITLE "Oh My Golly!" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\11 - Pixies - Vamos.wav" WAVE + TRACK 11 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 + INDEX 02 00:44:70 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\12 - Pixies - I'm Amazed.wav" WAVE + TRACK 12 AUDIO + TITLE "I'm Amazed" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\13 - Pixies - Brick is Red.wav" WAVE + TRACK 13 AUDIO + TITLE "Brick is Red" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\14 - Pixies - Caribou.wav" WAVE + TRACK 14 AUDIO + TITLE "Caribou" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\15 - Pixies - Vamos.wav" WAVE + TRACK 15 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\16 - Pixies - Isla de Encanta.wav" WAVE + TRACK 16 AUDIO + TITLE "Isla de Encanta" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\17 - Pixies - Ed is Dead.wav" WAVE + TRACK 17 AUDIO + TITLE "Ed is Dead" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\18 - Pixies - The Holyday Song.wav" WAVE + TRACK 18 AUDIO + TITLE "The Holyday Song" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\19 - Pixies - Nimrod's Son.wav" WAVE + TRACK 19 AUDIO + TITLE "Nimrod's Son" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\20 - Pixies - I've Been Tired.wav" WAVE + TRACK 20 AUDIO + TITLE "I've Been Tired" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\21 - Pixies - Levitate Me.wav" WAVE + TRACK 21 AUDIO + TITLE "Levitate Me" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 diff --git a/morituri/test/surferrosa.eac.noncompliant.cue b/morituri/test/surferrosa.eac.noncompliant.cue new file mode 100644 index 0000000..9e7f189 --- /dev/null +++ b/morituri/test/surferrosa.eac.noncompliant.cue @@ -0,0 +1,136 @@ +REM GENRE Alternative +REM DATE 1987 +REM DISCID 350CAA15 +REM COMMENT "ExactAudioCopy v0.99pb4" +CATALOG 0000000000000 +PERFORMER "Pixies" +TITLE "Surfer Rosa & Come on Pilgrim" +FILE "Pixies - Surfer Rosa & Come on Pilgrim\01 - Pixies - Bone Machine.wav" WAVE + TRACK 01 AUDIO + TITLE "Bone Machine" + PERFORMER "Pixies" + ISRC 000000000000 + PREGAP 00:00:32 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\02 - Pixies - Break My Body.wav" WAVE + TRACK 02 AUDIO + TITLE "Break My Body" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 + TRACK 03 AUDIO + TITLE "Something Against You" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 00 02:05:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\03 - Pixies - Something Against You.wav" WAVE + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\04 - Pixies - Broken Face.wav" WAVE + TRACK 04 AUDIO + TITLE "Broken Face" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\05 - Pixies - Gigantic.wav" WAVE + TRACK 05 AUDIO + TITLE "Gigantic" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\06 - Pixies - River Euphrates.wav" WAVE + TRACK 06 AUDIO + TITLE "River Euphrates" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\07 - Pixies - Where Is My Mind .wav" WAVE + TRACK 07 AUDIO + TITLE "Where Is My Mind?" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\08 - Pixies - Cactus.wav" WAVE + TRACK 08 AUDIO + TITLE "Cactus" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\09 - Pixies - Tony's Theme.wav" WAVE + TRACK 09 AUDIO + TITLE "Tony's Theme" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\10 - Pixies - Oh My Golly!.wav" WAVE + TRACK 10 AUDIO + TITLE "Oh My Golly!" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\11 - Pixies - Vamos.wav" WAVE + TRACK 11 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 + INDEX 02 00:44:70 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\12 - Pixies - I'm Amazed.wav" WAVE + TRACK 12 AUDIO + TITLE "I'm Amazed" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\13 - Pixies - Brick is Red.wav" WAVE + TRACK 13 AUDIO + TITLE "Brick is Red" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\14 - Pixies - Caribou.wav" WAVE + TRACK 14 AUDIO + TITLE "Caribou" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\15 - Pixies - Vamos.wav" WAVE + TRACK 15 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\16 - Pixies - Isla de Encanta.wav" WAVE + TRACK 16 AUDIO + TITLE "Isla de Encanta" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\17 - Pixies - Ed is Dead.wav" WAVE + TRACK 17 AUDIO + TITLE "Ed is Dead" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\18 - Pixies - The Holyday Song.wav" WAVE + TRACK 18 AUDIO + TITLE "The Holyday Song" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\19 - Pixies - Nimrod's Son.wav" WAVE + TRACK 19 AUDIO + TITLE "Nimrod's Son" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\20 - Pixies - I've Been Tired.wav" WAVE + TRACK 20 AUDIO + TITLE "I've Been Tired" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 +FILE "Pixies - Surfer Rosa & Come on Pilgrim\21 - Pixies - Levitate Me.wav" WAVE + TRACK 21 AUDIO + TITLE "Levitate Me" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 00:00:00 diff --git a/morituri/test/surferrosa.eac.single.cue b/morituri/test/surferrosa.eac.single.cue new file mode 100644 index 0000000..702eb48 --- /dev/null +++ b/morituri/test/surferrosa.eac.single.cue @@ -0,0 +1,116 @@ +REM GENRE Alternative +REM DATE 1987 +REM DISCID 350CAA15 +REM COMMENT "ExactAudioCopy v0.99pb4" +CATALOG 0000000000000 +PERFORMER "Pixies" +TITLE "Surfer Rosa & Come on Pilgrim" +FILE "Range.wav" WAVE + TRACK 01 AUDIO + TITLE "Bone Machine" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 00 00:00:00 + INDEX 01 00:00:32 + TRACK 02 AUDIO + TITLE "Break My Body" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 03:03:42 + TRACK 03 AUDIO + TITLE "Something Against You" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 00 05:08:42 + INDEX 01 05:09:12 + TRACK 04 AUDIO + TITLE "Broken Face" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 06:56:67 + TRACK 05 AUDIO + TITLE "Gigantic" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 08:27:00 + TRACK 06 AUDIO + TITLE "River Euphrates" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 12:21:70 + TRACK 07 AUDIO + TITLE "Where Is My Mind?" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 14:53:60 + TRACK 08 AUDIO + TITLE "Cactus" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 18:47:15 + TRACK 09 AUDIO + TITLE "Tony's Theme" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 21:03:70 + TRACK 10 AUDIO + TITLE "Oh My Golly!" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 22:56:15 + TRACK 11 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 24:43:32 + INDEX 02 25:28:27 + TRACK 12 AUDIO + TITLE "I'm Amazed" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 29:49:20 + TRACK 13 AUDIO + TITLE "Brick is Red" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 31:31:27 + TRACK 14 AUDIO + TITLE "Caribou" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 33:32:20 + TRACK 15 AUDIO + TITLE "Vamos" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 36:46:45 + TRACK 16 AUDIO + TITLE "Isla de Encanta" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 39:40:22 + TRACK 17 AUDIO + TITLE "Ed is Dead" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 41:21:47 + TRACK 18 AUDIO + TITLE "The Holyday Song" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 43:51:47 + TRACK 19 AUDIO + TITLE "Nimrod's Son" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 46:06:10 + TRACK 20 AUDIO + TITLE "I've Been Tired" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 48:23:25 + TRACK 21 AUDIO + TITLE "Levitate Me" + PERFORMER "Pixies" + ISRC 000000000000 + INDEX 01 51:24:07 From 7118555a7f000918e168d6ee835f2edcff9fc9b0 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 21:44:36 +0200 Subject: [PATCH 092/132] add doc --- morituri/image/toc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 4190324..22649ad 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -319,6 +319,10 @@ class TocFile(object, log.Loggable): self._messages.append((number + 1, message)) def getTrackLength(self, track): + """ + Returns the length of the given track, from its INDEX 01 to the next + track's INDEX 01 + """ # returns track length in frames, or -1 if can't be determined and # complete file should be assumed # FIXME: this assumes a track can only be in one file; is this true ? From f11672fd3de1df52cf37f77281d1645e07e76bfa Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 21:53:32 +0200 Subject: [PATCH 093/132] add a Source object, but don't use it yet --- morituri/image/toc.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 22649ad..1b7d29f 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -88,6 +88,38 @@ _INDEX_RE = re.compile(r""" """, re.VERBOSE) +class Sources(log.Loggable): + """ + I represent the list of sources used in the .toc file. + Each SILENCE and each FILE is a source. + If the filename for FILE doesn't change, the counter is not increased. + """ + + def __init__(self): + self._sources = [] + + def append(self, counter, offset, source): + """ + @param counter: the source counter; updates for each different + data source (silence or different file path) + @type counter: int + @param offset: the absolute disc offset where this source starts + """ + self.debug('Appending source, counter %d, offset %d, source %r' % ( + counter, offset, source)) + self._sources.append((counter, offset, source)) + + def get(self, offset): + """ + Retrieve the source used at the given offset. + """ + for i, (c, o, s) in enumerate(self._sources): + if offset < o: + return self._sources[i - 1] + + return self._sources[-1] + + class TocFile(object, log.Loggable): def __init__(self, path): @@ -119,6 +151,7 @@ class TocFile(object, log.Loggable): totalLength = 0 # accrued during TRACK record parsing, total disc pregapLength = 0 # length of the pre-gap, current track in for loop + sources = Sources() # the first track's INDEX 1 can only be gotten from the .toc # file once the first pregap is calculated; so we add INDEX 1 @@ -206,6 +239,7 @@ class TocFile(object, log.Loggable): if m: length = m.group('length') self.debug('SILENCE of %r', length) + sources.append(counter, absoluteOffset, None) if currentFile is not None: self.debug('SILENCE after FILE, increasing counter') counter += 1 @@ -241,6 +275,8 @@ class TocFile(object, log.Loggable): trackNumber, counter) currentFile = File(filePath, common.msfToFrames(start), common.msfToFrames(length)) + sources.append(counter, absoluteOffset + currentLength, + currentFile) #absoluteOffset += common.msfToFrames(start) currentLength += common.msfToFrames(length) From 73a7056d85f8458791b9644570babfae60ccf948 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 21:58:17 +0200 Subject: [PATCH 094/132] handle index 00 of track 1 specially --- morituri/image/table.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/morituri/image/table.py b/morituri/image/table.py index 9ea055c..79d1422 100644 --- a/morituri/image/table.py +++ b/morituri/image/table.py @@ -547,11 +547,19 @@ class Table(object, log.Loggable): if key in self.cdtext: lines.append('%s "%s"' % (key, self.cdtext[key])) - # add the first FILE line - path = self.tracks[0].getFirstIndex().path - counter = self.tracks[0].getFirstIndex().counter + # add the first FILE line; EAC always puts the first FILE + # statement before TRACK 01 + firstTrack = self.tracks[0] + indexOne = firstTrack.getIndex(1) + + firstIndex = firstTrack.getFirstIndex() + path = indexOne.path + counter = indexOne.counter + + assert path, "No path on TRACK 01 INDEX 01" writeFile(path) + for i, track in enumerate(self.tracks): # FIXME: skip data tracks for now if not track.audio: @@ -577,6 +585,14 @@ class Table(object, log.Loggable): for number in indexes: index = track.indexes[number] + # handle TRACK 01 INDEX 00 specially + if i == 0 and number == 0: + # if we have a silent pre-gap, output it + if not index.path: + length = indexOne.absolute - index.absolute + lines.append(" PREGAP %s" % common.framesToMSF(length)) + continue + if index.counter != counter: writeFile(index.path) counter = index.counter From 1ac8d43f88d62b918aea8443f085914f90396e3b Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 23:16:03 +0200 Subject: [PATCH 095/132] use diffStrings --- morituri/test/test_image_toc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index e41bc73..0ae3e90 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -139,7 +139,7 @@ class BlocTestCase(common.TestCase): cue = self.toc.table.cue() ref = open(os.path.join(os.path.dirname(__file__), 'bloc.cue')).read() - self.assertEquals(cue, ref) + common.diffStrings(cue, ref) def testCDDBId(self): self.toc.table.absolutize() From 142fa9e33eb0589205e7280ef862dd7bf2797211 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 21:59:43 +0200 Subject: [PATCH 096/132] create index with the correct path by using sources --- morituri/image/toc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 1b7d29f..e0b4656 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -309,10 +309,11 @@ class TocFile(object, log.Loggable): continue length = common.msfToFrames(m.group('length')) - currentTrack.index(0, path=currentFile.path, + c, o, s = sources.get(absoluteOffset) + currentTrack.index(0, path=s and s.path or None, absolute=absoluteOffset, - relative=relativeOffset, counter=counter) - self.debug('track %d, added index %r', + relative=relativeOffset, counter=c) + self.debug('[track %2d index 00] added %r', currentTrack.number, currentTrack.getIndex(0)) # store the pregapLength to add it when we index 1 for this # track on the next iteration From d1940b639f10352cc08bb09185101c9449dfcef5 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 21:58:45 +0200 Subject: [PATCH 097/132] update tests --- morituri/test/test_image_toc.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 0ae3e90..a5bd4de 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -23,7 +23,10 @@ class CureTestCase(common.TestCase): def testGetTrackLength(self): t = self.toc.table.tracks[0] # first track has known length because the .toc is a single file - self.assertEquals(self.toc.getTrackLength(t), 28324) + # its length is all of track 1 from .toc, plus the INDEX 00 length + # of track 2 + self.assertEquals(self.toc.getTrackLength(t), + (((6 * 60) + 16) * 75 + 45) + ((1 * 75) + 4)) # last track has unknown length t = self.toc.table.tracks[-1] self.assertEquals(self.toc.getTrackLength(t), -1) @@ -403,10 +406,18 @@ class SurferRosaTestCase(common.TestCase): # HTOA t = self.toc.table.tracks[0] self.assertEquals(len(t.indexes), 2) - self.assertEquals(t.getIndex(0).relative, 0) - self.assertEquals(t.getIndex(0).absolute, 0) - self.assertEquals(t.getIndex(1).relative, 32) - self.assertEquals(t.getIndex(1).absolute, 32) + + i0 = t.getIndex(0) + self.assertEquals(i0.relative, 0) + self.assertEquals(i0.absolute, 0) + self.assertEquals(i0.path, None) + self.assertEquals(i0.counter, 0) + + i1 = t.getIndex(1) + self.assertEquals(i1.relative, 32) + self.assertEquals(i1.absolute, 32) + self.assertEquals(i1.path, 'data.wav') + self.assertEquals(i1.counter, 1) # track 11, Vamos From 2c234184904d22a9cc23fe6426a97f963e3a16ed Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 21:59:13 +0200 Subject: [PATCH 098/132] now take relative from currentFile --- morituri/image/toc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index e0b4656..9cc0c89 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -200,7 +200,7 @@ class TocFile(object, log.Loggable): # FIXME: why not set absolute offsets too ? currentTrack.index(1, path=currentFile.path, absolute=absoluteOffset + pregapLength, - relative=relativeOffset + pregapLength, + relative=currentFile.start + pregapLength, counter=counter) self.debug( '[track %02d index 01] pregapLength %r, added %r', @@ -339,7 +339,7 @@ class TocFile(object, log.Loggable): if currentTrack: currentTrack.index(1, path=currentFile.path, absolute=absoluteOffset + pregapLength, - relative=relativeOffset + pregapLength, counter=counter) + relative=currentFile.start, counter=counter) self.debug('[track %02d index 01] last track, added %r', currentTrack.number, currentTrack.getIndex(1)) From e3e3a87547f6617bc0877d7302d57d39a95b6ad1 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 19 Aug 2013 00:18:53 +0200 Subject: [PATCH 099/132] WIP: try to set relative offset correctly based on sources --- morituri/image/toc.py | 16 +++++++++++++++- morituri/test/test_image_toc.py | 13 ++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 9cc0c89..189de75 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -119,6 +119,16 @@ class Sources(log.Loggable): return self._sources[-1] + def getCounterStart(self, counter): + """ + Retrieve the absolute offset of the first source for this counter + """ + for i, (c, o, s) in enumerate(self._sources): + if c == counter: + return self._sources[i][1] + + return self._sources[-1][1] + class TocFile(object, log.Loggable): @@ -197,7 +207,6 @@ class TocFile(object, log.Loggable): # set index 1 of previous track if there was one, using # pregapLength if applicable if currentTrack: - # FIXME: why not set absolute offsets too ? currentTrack.index(1, path=currentFile.path, absolute=absoluteOffset + pregapLength, relative=currentFile.start + pregapLength, @@ -310,6 +319,11 @@ class TocFile(object, log.Loggable): length = common.msfToFrames(m.group('length')) c, o, s = sources.get(absoluteOffset) + self.debug('at abs offset %d, we are in source %r' % ( + absoluteOffset, s)) + counterStart = sources.getCounterStart(c) + relativeOffset = absoluteOffset - counterStart + currentTrack.index(0, path=s and s.path or None, absolute=absoluteOffset, relative=relativeOffset, counter=c) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index a5bd4de..2a451e6 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -130,9 +130,16 @@ class BlocTestCase(common.TestCase): self.assertEquals(self.toc.getTrackLength(t), -1) def testIndexes(self): - t = self.toc.table.tracks[0] - self.assertEquals(t.getIndex(0).relative, 0) - self.assertEquals(t.getIndex(1).relative, 15220) + firstTrack = self.toc.table.tracks[0] + index0 = firstTrack.getIndex(0) + self.assertEquals(index0.absolute, 0) + self.assertEquals(index0.relative, 0) + self.assertEquals(index0.counter, 0) + + index1 = firstTrack.getIndex(1) + self.assertEquals(index1.absolute, 15220) + self.assertEquals(index1.relative, 0) + self.assertEquals(index1.counter, 1) # This disc has a pre-gap, so is a good test for .CUE writing From 1ef45c2b2a1f52617530c3bb4f388c9c8c60a828 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 18 Aug 2013 21:45:18 +0200 Subject: [PATCH 100/132] stop absolutizing --- morituri/test/test_image_toc.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 2a451e6..45b369b 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -62,7 +62,7 @@ class CureTestCase(common.TestCase): self._assertAbsolute(2, 1, 28324) self._assertPath(1, 1, "data.wav") - self.toc.table.absolutize() + # self.toc.table.absolutize() self.toc.table.clearFiles() self._assertAbsolute(1, 1, 0) @@ -86,7 +86,7 @@ class CureTestCase(common.TestCase): self._assertRelative(2, 1, None) def testConvertCue(self): - self.toc.table.absolutize() + # self.toc.table.absolutize() cue = self.toc.table.cue() ref = open(os.path.join(os.path.dirname(__file__), 'cure.cue')).read( ).decode('utf-8') @@ -152,7 +152,7 @@ class BlocTestCase(common.TestCase): common.diffStrings(cue, ref) def testCDDBId(self): - self.toc.table.absolutize() + # self.toc.table.absolutize() # cd-discid output: # ad0be00d 13 15370 35019 51532 69190 84292 96826 112527 132448 # 148595 168072 185539 203331 222103 3244 @@ -161,7 +161,7 @@ class BlocTestCase(common.TestCase): def testAccurateRip(self): # we verify it because it has failed in readdisc in the past - self.toc.table.absolutize() + # self.toc.table.absolutize() self.assertEquals(self.toc.table.getAccurateRipURL(), 'http://www.accuraterip.com/accuraterip/' 'e/d/2/dBAR-013-001af2de-0105994e-ad0be00d.bin') @@ -189,7 +189,7 @@ class BreedersTestCase(common.TestCase): self.assertEquals(cdt['TITLE'], 'OVERGLAZED') def testConvertCue(self): - self.toc.table.absolutize() + # self.toc.table.absolutize() self.failUnless(self.toc.table.hasTOC()) cue = self.toc.table.cue() ref = open(os.path.join(os.path.dirname(__file__), @@ -211,7 +211,7 @@ class LadyhawkeTestCase(common.TestCase): self.failIf(self.toc.table.tracks[-1].audio) def testCDDBId(self): - self.toc.table.absolutize() + #self.toc.table.absolutize() self.assertEquals(self.toc.table.getCDDBDiscId(), 'c60af50d') # output from cd-discid: # c60af50d 13 150 15687 31841 51016 66616 81352 99559 116070 133243 @@ -260,7 +260,7 @@ class CapitalMergeTestCase(common.TestCase): self.table.merge(self.toc2.table) def testCDDBId(self): - self.table.absolutize() + #self.table.absolutize() self.assertEquals(self.table.getCDDBDiscId(), 'b910140c') # output from cd-discid: # b910140c 12 24320 44855 64090 77885 88095 104020 118245 129255 141765 @@ -327,7 +327,7 @@ class TOTBLTestCase(common.TestCase): self.assertEquals(len(self.toc.table.tracks), 11) def testCDDBId(self): - self.toc.table.absolutize() + #self.toc.table.absolutize() self.assertEquals(self.toc.table.getCDDBDiscId(), '810b7b0b') @@ -437,7 +437,7 @@ class SurferRosaTestCase(common.TestCase): self.assertEquals(t.getIndex(2).relative, 3370) self.assertEquals(t.getIndex(2).absolute, None) - self.toc.table.absolutize() + # self.toc.table.absolutize() self.assertEquals(t.getIndex(2).absolute, 3370) # print self.toc.table.cue() From 2200776b30c94ab68f04166c1e94799a62da9e8b Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:07:49 +0200 Subject: [PATCH 101/132] update debug --- morituri/image/toc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 189de75..2363dd8 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -105,7 +105,7 @@ class Sources(log.Loggable): @type counter: int @param offset: the absolute disc offset where this source starts """ - self.debug('Appending source, counter %d, offset %d, source %r' % ( + self.debug('Appending source, counter %d, abs offset %d, source %r' % ( counter, offset, source)) self._sources.append((counter, offset, source)) From 38193c7f0ca4000accb210ab1ccc91c949c87753 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:08:09 +0200 Subject: [PATCH 102/132] add/fix debug --- morituri/image/toc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 2363dd8..04ac542 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -327,7 +327,7 @@ class TocFile(object, log.Loggable): currentTrack.index(0, path=s and s.path or None, absolute=absoluteOffset, relative=relativeOffset, counter=c) - self.debug('[track %2d index 00] added %r', + self.debug('[track %02d index 00] added %r', currentTrack.number, currentTrack.getIndex(0)) # store the pregapLength to add it when we index 1 for this # track on the next iteration From a1e787648e42929c5ea56674c276da98fc49b083 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:08:42 +0200 Subject: [PATCH 103/132] add more tests, after calculating expected values for this disc --- morituri/test/test_image_toc.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 45b369b..f5241f8 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -130,16 +130,28 @@ class BlocTestCase(common.TestCase): self.assertEquals(self.toc.getTrackLength(t), -1) def testIndexes(self): - firstTrack = self.toc.table.tracks[0] - index0 = firstTrack.getIndex(0) - self.assertEquals(index0.absolute, 0) - self.assertEquals(index0.relative, 0) - self.assertEquals(index0.counter, 0) + track01 = self.toc.table.tracks[0] + index00 = track01.getIndex(0) + self.assertEquals(index00.absolute, 0) + self.assertEquals(index00.relative, 0) + self.assertEquals(index00.counter, 0) - index1 = firstTrack.getIndex(1) - self.assertEquals(index1.absolute, 15220) - self.assertEquals(index1.relative, 0) - self.assertEquals(index1.counter, 1) + index01 = track01.getIndex(1) + self.assertEquals(index01.absolute, 15220) + self.assertEquals(index01.relative, 0) + self.assertEquals(index01.counter, 1) + + track05 = self.toc.table.tracks[4] + + index00 = track05.getIndex(0) + self.assertEquals(index00.absolute, 84070) + self.assertEquals(index00.relative, 68850) + self.assertEquals(index00.counter, 1) + + index01 = track05.getIndex(1) + self.assertEquals(index01.absolute, 84142) + self.assertEquals(index01.relative, 68922) + self.assertEquals(index01.counter, 1) # This disc has a pre-gap, so is a good test for .CUE writing From b59bf8f98a2b68c2349fd939c4e7965eda2de32a Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:10:02 +0200 Subject: [PATCH 104/132] make sources an object property --- morituri/image/toc.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 04ac542..52e5ade 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -142,6 +142,9 @@ class TocFile(object, log.Loggable): self.table = table.Table() self.logName = '' % id(self) + self._sources = Sources() + + def parse(self): # these two objects start as None then get set as real objects, # so no need to complain about them here @@ -161,8 +164,6 @@ class TocFile(object, log.Loggable): totalLength = 0 # accrued during TRACK record parsing, total disc pregapLength = 0 # length of the pre-gap, current track in for loop - sources = Sources() - # the first track's INDEX 1 can only be gotten from the .toc # file once the first pregap is calculated; so we add INDEX 1 # at the end of each parsed TRACK record @@ -248,7 +249,7 @@ class TocFile(object, log.Loggable): if m: length = m.group('length') self.debug('SILENCE of %r', length) - sources.append(counter, absoluteOffset, None) + self._sources.append(counter, absoluteOffset, None) if currentFile is not None: self.debug('SILENCE after FILE, increasing counter') counter += 1 @@ -284,7 +285,7 @@ class TocFile(object, log.Loggable): trackNumber, counter) currentFile = File(filePath, common.msfToFrames(start), common.msfToFrames(length)) - sources.append(counter, absoluteOffset + currentLength, + self._sources.append(counter, absoluteOffset + currentLength, currentFile) #absoluteOffset += common.msfToFrames(start) currentLength += common.msfToFrames(length) @@ -318,10 +319,10 @@ class TocFile(object, log.Loggable): continue length = common.msfToFrames(m.group('length')) - c, o, s = sources.get(absoluteOffset) + c, o, s = self._sources.get(absoluteOffset) self.debug('at abs offset %d, we are in source %r' % ( absoluteOffset, s)) - counterStart = sources.getCounterStart(c) + counterStart = self._sources.getCounterStart(c) relativeOffset = absoluteOffset - counterStart currentTrack.index(0, path=s and s.path or None, From ddea002a8f443a803620614cbc207517c470cfc2 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:32:06 +0200 Subject: [PATCH 105/132] also add source for DATAFILE --- morituri/image/toc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 52e5ade..7d293fc 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -306,6 +306,8 @@ class TocFile(object, log.Loggable): trackNumber, counter) # FIXME: assume that a MODE2_FORM_MIX track always starts at 0 currentFile = File(filePath, 0, common.msfToFrames(length)) + self._sources.append(counter, absoluteOffset + currentLength, + currentFile) #absoluteOffset += common.msfToFrames(start) currentLength += common.msfToFrames(length) From 61fdb2c332067c9b216322b491f6920b28abe747 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:38:19 +0200 Subject: [PATCH 106/132] move some assignments around to be more logical --- morituri/image/toc.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 7d293fc..62c5c52 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -217,24 +217,27 @@ class TocFile(object, log.Loggable): currentTrack.number, pregapLength, currentTrack.getIndex(1)) - # update running totals + # create a new track to be filled by later lines trackNumber += 1 + trackMode = m.group('mode') + audio = trackMode == 'AUDIO' + currentTrack = table.Track(trackNumber, audio=audio) + self.table.tracks.append(currentTrack) + + # update running totals absoluteOffset += currentLength relativeOffset += currentLength totalLength += currentLength - trackMode = m.group('mode') - - # reset counters - currentLength = 0 - indexNumber = 1 - pregapLength = 0 # FIXME: track mode self.debug('found track %d, mode %s, at absoluteOffset %d', trackNumber, trackMode, absoluteOffset) - audio = trackMode == 'AUDIO' - currentTrack = table.Track(trackNumber, audio=audio) - self.table.tracks.append(currentTrack) + + # reset counters relative to a track + currentLength = 0 + indexNumber = 1 + pregapLength = 0 + continue # look for ISRC lines From bcd350f7686a0bf866d98999d1f5c9bacca7471d Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:38:40 +0200 Subject: [PATCH 107/132] use a common _index method for all cases --- morituri/image/toc.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/morituri/image/toc.py b/morituri/image/toc.py index 62c5c52..c83e940 100644 --- a/morituri/image/toc.py +++ b/morituri/image/toc.py @@ -144,6 +144,24 @@ class TocFile(object, log.Loggable): self._sources = Sources() + def _index(self, currentTrack, i, absoluteOffset, trackOffset): + absolute = absoluteOffset + trackOffset + # this may be in a new source, so calculate relative + c, o, s = self._sources.get(absolute) + self.debug('at abs offset %d, we are in source %r' % ( + absolute, s)) + counterStart = self._sources.getCounterStart(c) + relative = absolute - counterStart + + currentTrack.index(i, path=s.path, + absolute=absolute, + relative=relative, + counter=c) + self.debug( + '[track %02d index %02d] trackOffset %r, added %r', + currentTrack.number, i, trackOffset, + currentTrack.getIndex(i)) + def parse(self): # these two objects start as None then get set as real objects, @@ -208,14 +226,7 @@ class TocFile(object, log.Loggable): # set index 1 of previous track if there was one, using # pregapLength if applicable if currentTrack: - currentTrack.index(1, path=currentFile.path, - absolute=absoluteOffset + pregapLength, - relative=currentFile.start + pregapLength, - counter=counter) - self.debug( - '[track %02d index 01] pregapLength %r, added %r', - currentTrack.number, pregapLength, - currentTrack.getIndex(1)) + self._index(currentTrack, 1, absoluteOffset, pregapLength) # create a new track to be filled by later lines trackNumber += 1 @@ -349,19 +360,11 @@ class TocFile(object, log.Loggable): indexNumber += 1 offset = common.msfToFrames(m.group('offset')) - currentTrack.index(indexNumber, path=currentFile.path, - relative=offset, counter=counter) - self.debug('[track %02d index %02d] added %r', - currentTrack.number, indexNumber, - currentTrack.getIndex(indexNumber)) + self._index(currentTrack, indexNumber, absoluteOffset, offset) # handle index 1 of final track, if any if currentTrack: - currentTrack.index(1, path=currentFile.path, - absolute=absoluteOffset + pregapLength, - relative=currentFile.start, counter=counter) - self.debug('[track %02d index 01] last track, added %r', - currentTrack.number, currentTrack.getIndex(1)) + self._index(currentTrack, 1, absoluteOffset, pregapLength) # totalLength was added up to the penultimate track self.table.leadout = totalLength + currentLength From bf3db822dd0ef6dbbefa6293687eeda0ad9ee4d9 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 00:43:35 +0200 Subject: [PATCH 108/132] strokes test case is now correct --- morituri/test/test_image_toc.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index f5241f8..803040b 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -360,17 +360,11 @@ class StrokesTestCase(common.TestCase): i0 = t.getIndex(0) self.assertEquals(i0.relative, 0) self.assertEquals(i0.absolute, 0) - # FIXME: this is what we should have - # self.assertEquals(i0.counter, 0) - # self.assertEquals(i0.path, None) - # FIXME: this is what it is right now - self.assertEquals(i0.counter, 1) - self.assertEquals(i0.path, u'data.wav') + self.assertEquals(i0.counter, 0) + self.assertEquals(i0.path, None) i1 = t.getIndex(1) - # FIXME: relative should be 0, as it should point to the start - # of data.wav - self.assertEquals(i1.relative, 1) + self.assertEquals(i1.relative, 0) self.assertEquals(i1.absolute, 1) self.assertEquals(i1.counter, 1) self.assertEquals(i1.path, u'data.wav') @@ -378,9 +372,7 @@ class StrokesTestCase(common.TestCase): cue = self._filterCue(self.toc.table.cue()) ref = self._filterCue(open(os.path.join(os.path.dirname(__file__), 'strokes-someday.eac.cue')).read()) - # FIXME: this diff should match - # common.diffStrings(cue, ref) - self.assertRaises(AssertionError, common.diffStrings, cue, ref) + common.diffStrings(cue, ref) def _filterCue(self, output): # helper to be able to compare our generated .cue with the From 9f2ff9ee52e771bd147b11527e97908ade75834a Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 01:05:33 +0200 Subject: [PATCH 109/132] fix expectations on surfer rosa test looks like it was interpreting relative on an index02 relative to the track's index 00, instead of relative to the source --- morituri/test/test_image_toc.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 803040b..49d56b8 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -402,6 +402,9 @@ class StrokesTestCase(common.TestCase): # Surfer Rosa has # track 00 consisting of 32 frames of SILENCE # track 11 Vamos with an INDEX 02 +# compared to an EAC single .cue file, all our offsets are 32 frames off +# because the toc uses silence for track 01 index 00 while EAC puts it in +# Range.wav class SurferRosaTestCase(common.TestCase): @@ -425,7 +428,7 @@ class SurferRosaTestCase(common.TestCase): self.assertEquals(i0.counter, 0) i1 = t.getIndex(1) - self.assertEquals(i1.relative, 32) + self.assertEquals(i1.relative, 0) self.assertEquals(i1.absolute, 32) self.assertEquals(i1.path, 'data.wav') self.assertEquals(i1.counter, 1) @@ -436,13 +439,10 @@ class SurferRosaTestCase(common.TestCase): self.assertEquals(len(t.indexes), 2) # 32 frames of silence, and 1483 seconds of data.wav - self.assertEquals(t.getIndex(1).relative, 111257) + self.assertEquals(t.getIndex(1).relative, 111225) self.assertEquals(t.getIndex(1).absolute, 111257) - self.assertEquals(t.getIndex(2).relative, 3370) - self.assertEquals(t.getIndex(2).absolute, None) - - # self.toc.table.absolutize() - self.assertEquals(t.getIndex(2).absolute, 3370) + self.assertEquals(t.getIndex(2).relative, 111225 + 3370) + self.assertEquals(t.getIndex(2).absolute, 111257 + 3370) # print self.toc.table.cue() From 38554e52246d44f75fb42a2e6a5047c52e4e611d Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 01:12:32 +0200 Subject: [PATCH 110/132] add note --- HACKING | 1 + 1 file changed, 1 insertion(+) diff --git a/HACKING b/HACKING index cc25424..5152f78 100644 --- a/HACKING +++ b/HACKING @@ -44,3 +44,4 @@ PLEXTOR CD-R PX-W8432T Read offset of device is: 355. Test discs ---------- The Strokes - Someday (promo): has 1 frame silence marked as SILENCE +The Pixies - Surfer Rosa/Come on Pilgrim: has pre-gap, and INDEX 02 on TRACK 11 From ea9225a9de6f2b8e04e07da425476c97a98273ef Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Tue, 20 Aug 2013 01:16:45 +0200 Subject: [PATCH 111/132] now that we have a literal conversion of the toc, update the toc doesn't notice the real pregap audio data, so marks the HTOA as silence, and puts all following audio data in one file. Update .cue file to reflect this; ie, create a cue file that misses the HTOA --- morituri/test/bloc.cue | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/morituri/test/bloc.cue b/morituri/test/bloc.cue index 2176f26..a6c3295 100644 --- a/morituri/test/bloc.cue +++ b/morituri/test/bloc.cue @@ -2,37 +2,37 @@ REM DISCID AD0BE00D REM COMMENT "Morituri" FILE "data.wav" WAVE TRACK 01 AUDIO - INDEX 00 00:00:00 - INDEX 01 03:22:70 + PREGAP 03:22:70 + INDEX 01 00:00:00 TRACK 02 AUDIO - INDEX 01 07:44:69 + INDEX 01 04:21:74 TRACK 03 AUDIO - INDEX 01 11:25:07 + INDEX 01 08:02:12 TRACK 04 AUDIO - INDEX 01 15:20:40 + INDEX 01 11:57:45 TRACK 05 AUDIO - INDEX 00 18:40:70 - INDEX 01 18:41:67 + INDEX 00 15:18:00 + INDEX 01 15:18:72 TRACK 06 AUDIO - INDEX 00 21:28:35 - INDEX 01 21:29:01 + INDEX 00 18:05:40 + INDEX 01 18:06:06 TRACK 07 AUDIO - INDEX 00 24:58:10 - INDEX 01 24:58:27 + INDEX 00 21:35:15 + INDEX 01 21:35:32 TRACK 08 AUDIO - INDEX 00 29:23:69 - INDEX 01 29:23:73 + INDEX 00 26:00:74 + INDEX 01 26:01:03 TRACK 09 AUDIO - INDEX 00 32:59:09 - INDEX 01 32:59:20 + INDEX 00 29:36:14 + INDEX 01 29:36:25 TRACK 10 AUDIO - INDEX 01 37:18:72 + INDEX 01 33:56:02 TRACK 11 AUDIO - INDEX 00 41:11:21 - INDEX 01 41:11:64 + INDEX 00 37:48:26 + INDEX 01 37:48:69 TRACK 12 AUDIO - INDEX 00 45:07:40 - INDEX 01 45:09:06 + INDEX 00 41:44:45 + INDEX 01 41:46:11 TRACK 13 AUDIO - INDEX 00 49:19:06 - INDEX 01 49:19:28 + INDEX 00 45:56:11 + INDEX 01 45:56:33 From abc03e8e433dc3327c65882b6a3e20b9032c73c8 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 01:25:06 +0200 Subject: [PATCH 112/132] since the cached results have wrong (negative) relative for index > 01, make sure we don't use the cache by upping the version --- morituri/common/cache.py | 17 ++++++++++++----- morituri/image/table.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/morituri/common/cache.py b/morituri/common/cache.py index b1cdb06..d2ff9fe 100644 --- a/morituri/common/cache.py +++ b/morituri/common/cache.py @@ -115,7 +115,7 @@ class Persister(log.Loggable): os.unlink(self._path) -class PersistedCache(object): +class PersistedCache(log.Loggable): """ I wrap a directory of persisted objects. """ @@ -138,11 +138,15 @@ class PersistedCache(object): Returns the persister for the given key. """ persister = Persister(self._getPath(key)) + if persister.object: + if hasattr(persister.object, 'instanceVersion'): + o = persister.object + if o.instanceVersion < o.__class__.classVersion: + self.debug( + 'key %r persisted object version %d is outdated', + key, o.instanceVersion) + persister.object = None # FIXME: don't delete old objects atm - # if persister.object: - # if hasattr(persister.object, 'instanceVersion'): - # o = persister.object - # if o.instanceVersion < o.__class__.classVersion: # persister.delete() return persister @@ -230,6 +234,9 @@ class TableCache(log.Loggable): self.debug('cached table is for different mb id %r' % ( ptable.object.getMusicBrainzDiscId())) ptable.object = None + else: + self.debug('no valid cached table found for %r' % + cddbdiscid) if not ptable.object: # get an empty persistable from the writable location diff --git a/morituri/image/table.py b/morituri/image/table.py index 79d1422..56e33c5 100644 --- a/morituri/image/table.py +++ b/morituri/image/table.py @@ -168,7 +168,7 @@ class Table(object, log.Loggable): catalog = None # catalog number; FIXME: is this UPC ? cdtext = None - classVersion = 3 + classVersion = 4 def __init__(self, tracks=None): if not tracks: From 1141f5e27c3b0bb91992a0510645ef8ac9b7ae80 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 01:25:43 +0200 Subject: [PATCH 113/132] add type doc --- morituri/image/table.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/morituri/image/table.py b/morituri/image/table.py index 56e33c5..0386d49 100644 --- a/morituri/image/table.py +++ b/morituri/image/table.py @@ -635,6 +635,9 @@ class Table(object, log.Loggable): to adjust the path. Assumes all indexes have an absolute offset and will raise if not. + + @type track: C{int} + @type index: C{int} """ self.debug('setFile: track %d, index %d, path %r, ' 'length %r, counter %r', track, index, path, length, counter) From 19fa97e23aeb0343fe9fb8481f1428d0553fa11f Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 13:57:56 +0200 Subject: [PATCH 114/132] use more diffstrings; use it from reference to generated --- morituri/test/test_image_cue.py | 6 ++++-- morituri/test/test_image_toc.py | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/morituri/test/test_image_cue.py b/morituri/test/test_image_cue.py index b6e2909..5abfcd7 100644 --- a/morituri/test/test_image_cue.py +++ b/morituri/test/test_image_cue.py @@ -7,6 +7,8 @@ import unittest from morituri.image import table, cue +from morituri.test import common + class KingsSingleTestCase(unittest.TestCase): @@ -73,7 +75,7 @@ class WriteCueFileTestCase(unittest.TestCase): it.absolutize() it.leadout = 3000 - self.assertEquals(it.cue(), """REM DISCID 0C002802 + common.diffStrings(u"""REM DISCID 0C002802 REM COMMENT "Morituri" FILE "track01.wav" WAVE TRACK 01 AUDIO @@ -82,5 +84,5 @@ FILE "track01.wav" WAVE INDEX 00 00:13:25 FILE "track02.wav" WAVE INDEX 01 00:00:00 -""") +""", it.cue()) os.unlink(path) diff --git a/morituri/test/test_image_toc.py b/morituri/test/test_image_toc.py index 49d56b8..998bc0a 100644 --- a/morituri/test/test_image_toc.py +++ b/morituri/test/test_image_toc.py @@ -90,7 +90,7 @@ class CureTestCase(common.TestCase): cue = self.toc.table.cue() ref = open(os.path.join(os.path.dirname(__file__), 'cure.cue')).read( ).decode('utf-8') - common.diffStrings(cue, ref) + common.diffStrings(ref, cue) # we verify it because it has failed in readdisc in the past self.assertEquals(self.toc.table.getAccurateRipURL(), @@ -160,8 +160,8 @@ class BlocTestCase(common.TestCase): self.failUnless(self.toc.table.hasTOC()) cue = self.toc.table.cue() ref = open(os.path.join(os.path.dirname(__file__), - 'bloc.cue')).read() - common.diffStrings(cue, ref) + 'bloc.cue')).read().decode('utf-8') + common.diffStrings(ref, cue) def testCDDBId(self): # self.toc.table.absolutize() @@ -371,8 +371,8 @@ class StrokesTestCase(common.TestCase): cue = self._filterCue(self.toc.table.cue()) ref = self._filterCue(open(os.path.join(os.path.dirname(__file__), - 'strokes-someday.eac.cue')).read()) - common.diffStrings(cue, ref) + 'strokes-someday.eac.cue')).read()).decode('utf-8') + common.diffStrings(ref, cue) def _filterCue(self, output): # helper to be able to compare our generated .cue with the From 354bd7744f7dee547133499555b70d677656a9ad Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 24 Aug 2013 13:58:21 +0200 Subject: [PATCH 115/132] fix so first FILE goes before TRACK 01 and possible PREGAP --- morituri/image/table.py | 106 +++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/morituri/image/table.py b/morituri/image/table.py index 0386d49..c3cd570 100644 --- a/morituri/image/table.py +++ b/morituri/image/table.py @@ -523,11 +523,15 @@ class Table(object, log.Loggable): @rtype: C{unicode} """ + self.debug('generating .cue for cuePath %r', cuePath) + lines = [] def writeFile(path): targetPath = common.getRelativePath(path, cuePath) - lines.append('FILE "%s" WAVE' % targetPath) + line = 'FILE "%s" WAVE' % targetPath + lines.append(line) + self.debug('writeFile: %r' % line) # header main = ['PERFORMER', 'TITLE'] @@ -547,57 +551,91 @@ class Table(object, log.Loggable): if key in self.cdtext: lines.append('%s "%s"' % (key, self.cdtext[key])) + # FIXME: + # - the first FILE statement goes before the first TRACK, even if + # there is a non-file-using PREGAP + # - the following FILE statements come after the last INDEX that + # use that FILE; so before a next TRACK, PREGAP silence, ... + # add the first FILE line; EAC always puts the first FILE - # statement before TRACK 01 + # statement before TRACK 01 and any possible PRE-GAP firstTrack = self.tracks[0] + index = firstTrack.getFirstIndex() indexOne = firstTrack.getIndex(1) + counter = index.counter + track = firstTrack - firstIndex = firstTrack.getFirstIndex() - path = indexOne.path - counter = indexOne.counter - - assert path, "No path on TRACK 01 INDEX 01" - writeFile(path) + while not index.path: + t, i = self.getNextTrackIndex(track.number, index.number) + track = self.tracks[t - 1] + index = track.getIndex(i) + counter = index.counter + if index.path: + self.debug('counter %d, writeFile' % counter) + writeFile(index.path) for i, track in enumerate(self.tracks): + self.debug('track i %r, track %r' % (i, track)) # FIXME: skip data tracks for now if not track.audio: continue - # if there is no index 0, but there is a new file, advance - # FILE line here - if not 0 in track.indexes: - index = track.indexes[1] - if index.counter != counter: - writeFile(index.path) - counter = index.counter - lines.append(" TRACK %02d %s" % (i + 1, 'AUDIO')) - for key in CDTEXT_FIELDS: - if key in track.cdtext: - lines.append(' %s "%s"' % (key, track.cdtext[key])) - - if track.isrc is not None: - lines.append(" ISRC %s" % track.isrc) - indexes = track.indexes.keys() indexes.sort() + wroteTrack = False + for number in indexes: index = track.indexes[number] - # handle TRACK 01 INDEX 00 specially - if i == 0 and number == 0: - # if we have a silent pre-gap, output it - if not index.path: - length = indexOne.absolute - index.absolute - lines.append(" PREGAP %s" % common.framesToMSF(length)) - continue + self.debug('index %r, %r' % (number, index)) - if index.counter != counter: - writeFile(index.path) + # any time the source counter changes to a higher value, + # write a FILE statement + # it has to be higher, because we can run into the HTOA + # at counter 0 here + if index.counter > counter: + if index.path: + self.debug('counter %d, writeFile' % counter) + writeFile(index.path) + self.debug('setting counter to index.counter %r' % + index.counter) counter = index.counter - lines.append(" INDEX %02d %s" % (number, - common.framesToMSF(index.relative))) + + # any time we hit the first index, write a TRACK statement + if not wroteTrack: + wroteTrack = True + line = " TRACK %02d %s" % (i + 1, 'AUDIO') + lines.append(line) + self.debug('%r' % line) + + for key in CDTEXT_FIELDS: + if key in track.cdtext: + lines.append(' %s "%s"' % ( + key, track.cdtext[key])) + + if track.isrc is not None: + lines.append(" ISRC %s" % track.isrc) + + # handle TRACK 01 INDEX 00 specially + if 0 in indexes: + index00 = track.indexes[0] + if i == 0: + # if we have a silent pre-gap, output it + if not index00.path: + length = indexOne.absolute - index00.absolute + lines.append(" PREGAP %s" % + common.framesToMSF(length)) + continue + + # handle any other INDEX 00 after its TRACK + lines.append(" INDEX %02d %s" % (0, + common.framesToMSF(index00.relative))) + + if number > 0: + # index 00 is output after TRACK up above + lines.append(" INDEX %02d %s" % (number, + common.framesToMSF(index.relative))) lines.append("") From afd638fa4bd1b752cf00b230d1c6c74f3c146042 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 25 Aug 2013 17:18:04 +0200 Subject: [PATCH 116/132] add debug resultcache cue command to output a cue from a cached result --- morituri/rip/debug.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index 369be4c..00c1a12 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -25,6 +25,23 @@ from morituri.result import result from morituri.common import task, cache +class RCCue(logcommand.LogCommand): + + name = "cue" + summary = "write a cue file for the cached result" + + def do(self, args): + self._cache = cache.ResultCache() + + persisted = self._cache.getRipResult(args[0], create=False) + + if not persisted: + self.stderr.write( + 'Could not find a result for cddb disc id %s\n' % args[0]) + return 3 + + self.stdout.write(persisted.object.table.cue().encode('utf-8')) + class RCList(logcommand.LogCommand): @@ -85,14 +102,14 @@ class RCLog(logcommand.LogCommand): logger = klazz() self.stdout.write(logger.log(persisted.object).encode('utf-8')) - + class ResultCache(logcommand.LogCommand): summary = "debug result cache" aliases = ['rc', ] - subCommandClasses = [RCList, RCLog, ] + subCommandClasses = [RCCue, RCList, RCLog, ] class Checksum(logcommand.LogCommand): From f0a54d1cb6c3d157d54a465aec3612cf047ae36b Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 25 Aug 2013 17:41:10 +0200 Subject: [PATCH 117/132] add --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 992568d..3afb241 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,7 @@ TODO: +- after fixing relative, pregaps, and index 02, check when htoa is 0, + and add a setSilence to table to set a counter 0 with no path, and test + that the cue file puts a SILENCE/PREGAP - store drive features in a database - try http://www.ime.usp.br/~pjssilva/secure-cdparanoia.py and see if it is better at handling some bad cd's From fc282df8a6155dcac4fdb794e599ef02259a7c4c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Thu, 29 Aug 2013 19:50:03 +0200 Subject: [PATCH 118/132] start with a one frame interval until we know more if it's say four frames, we get to paused without eos, but then hit eos without level message --- morituri/common/encode.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/morituri/common/encode.py b/morituri/common/encode.py index 4dfdc32..27cce8f 100644 --- a/morituri/common/encode.py +++ b/morituri/common/encode.py @@ -179,14 +179,19 @@ class EncodeTask(ctask.GstPipelineTask): cgstreamer.removeAudioParsers() def getPipelineDesc(self): + # start with an emit interval of one frame, because we end up setting + # the final interval after paused and after processing some samples + # already, which is too late + interval = int(self.gst.SECOND / 75.0) return ''' filesrc location="%s" ! decodebin name=decoder ! audio/x-raw-int,width=16,depth=16,channels=2 ! - level name=level ! + level name=level interval=%d ! %s ! identity name=identity ! filesink location="%s" name=sink''' % ( gstreamer.quoteParse(self._inpath).encode('utf-8'), + interval, self._profile.pipeline, gstreamer.quoteParse(self._outpath).encode('utf-8')) From 1c47a4287dc8846677b71dc28f5ea761181d40a6 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Thu, 29 Aug 2013 19:50:51 +0200 Subject: [PATCH 119/132] add note --- HACKING | 1 + 1 file changed, 1 insertion(+) diff --git a/HACKING b/HACKING index 5152f78..586a564 100644 --- a/HACKING +++ b/HACKING @@ -45,3 +45,4 @@ Test discs ---------- The Strokes - Someday (promo): has 1 frame silence marked as SILENCE The Pixies - Surfer Rosa/Come on Pilgrim: has pre-gap, and INDEX 02 on TRACK 11 +Florence & The Machine - Lungs: data track From 2a2586f56f288050cf1fc38a295387abe30c0d30 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 15:49:58 -0500 Subject: [PATCH 120/132] allow all profiles, including lossy --- morituri/rip/debug.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index 00c1a12..47aa230 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -147,7 +147,7 @@ class Encode(logcommand.LogCommand): self.parser.add_option('', '--profile', action="store", dest="profile", help="profile for encoding (default '%s', choices '%s')" % ( - default, "', '".join(encode.PROFILES.keys())), + default, "', '".join(encode.ALL_PROFILES.keys())), default=default) def do(self, args): @@ -165,7 +165,7 @@ class Encode(logcommand.LogCommand): runner = task.SyncRunner() from morituri.common import encode - profile = encode.PROFILES[self.options.profile]() + profile = encode.ALL_PROFILES[self.options.profile]() self.debug('Encoding %s to %s', fromPath.encode('utf-8'), toPath.encode('utf-8')) From b379f60c2807e0408e5abf307308dbcfe2c3fc7c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 15:54:24 -0500 Subject: [PATCH 121/132] fix pychecker warning --- morituri/rip/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/morituri/rip/common.py b/morituri/rip/common.py index ccf1ef2..e931646 100644 --- a/morituri/rip/common.py +++ b/morituri/rip/common.py @@ -28,13 +28,13 @@ disc and track template are: ''' -def addTemplate(self): +def addTemplate(obj): # FIXME: get from config - self.parser.add_option('', '--track-template', + obj.parser.add_option('', '--track-template', action="store", dest="track_template", help="template for track file naming (default %default)", default=DEFAULT_TRACK_TEMPLATE) - self.parser.add_option('', '--disc-template', + obj.parser.add_option('', '--disc-template', action="store", dest="disc_template", help="template for disc file naming (default %default)", default=DEFAULT_DISC_TEMPLATE) From ff4e01fca3acb192b2935b94bc09bc12006f247e Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 15:54:52 -0500 Subject: [PATCH 122/132] fix another pychecker warning that was actually pointing out a bug --- morituri/rip/debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index 47aa230..678e64b 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -126,7 +126,7 @@ class Checksum(logcommand.LogCommand): from morituri.common import checksum for arg in args: - fromPath = unicode(args[0]) + fromPath = unicode(arg) checksumtask = checksum.CRC32Task(fromPath) From 02ae3d5bfd25d722642184c38e62f91c247b9b9c Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 16:01:20 -0500 Subject: [PATCH 123/132] add content to an older test --- morituri/test/test_common_common.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/morituri/test/test_common_common.py b/morituri/test/test_common_common.py index 073a93a..de9e8a4 100644 --- a/morituri/test/test_common_common.py +++ b/morituri/test/test_common_common.py @@ -55,5 +55,13 @@ class GetRealPathTestCase(tcommon.TestCase): fd, path = tempfile.mkstemp(suffix=u'back\\slash.flac') refPath = os.path.join(os.path.dirname(path), 'fake.cue') + self.assertEquals(common.getRealPath(refPath, path), + path) + + # same path, but with wav extension, will point to flac file + wavPath = path[:-4] + 'wav' + self.assertEquals(common.getRealPath(refPath, wavPath), + path) + os.close(fd) os.unlink(path) From 612ae766ae2c4046d72dadd9704dd1ab675dbff4 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 16:04:58 -0500 Subject: [PATCH 124/132] remove Rename operation currently unused and done on branch feature-rename --- morituri/rip/image.py | 55 ++++--------------------------------------- 1 file changed, 4 insertions(+), 51 deletions(-) diff --git a/morituri/rip/image.py b/morituri/rip/image.py index 6c766d6..a770f06 100644 --- a/morituri/rip/image.py +++ b/morituri/rip/image.py @@ -22,7 +22,7 @@ import os -from morituri.common import logcommand, accurip, program, encode, renamer +from morituri.common import logcommand, accurip, program from morituri.image import image from morituri.result import result @@ -110,6 +110,9 @@ class Retag(logcommand.LogCommand): def do(self, args): + # here to avoid import gst eating our options + from morituri.common import encode + prog = program.Program(self.getRootCommand().config, stdout=self.stdout) runner = task.SyncRunner() @@ -147,56 +150,6 @@ class Retag(logcommand.LogCommand): print '%s already tagged correctly' % path print -class Rename(logcommand.LogCommand): - - summary = "rename image and all files based on metadata" - - def addOptions(self): - self.parser.add_option('-R', '--release-id', - action="store", dest="release_id", - help="MusicBrainz release id to match to (if there are multiple)") - - - def do(self, args): - prog = program.Program(self.getRootCommand.config(), stdout=self.stdout) - runner = task.SyncRunner() - - for arg in args: - self.stdout.write('Renaming image %r\n' % arg) - arg = arg.decode('utf-8') - cueImage = image.Image(arg) - cueImage.setup(runner) - - mbdiscid = cueImage.table.getMusicBrainzDiscId() - - operator = renamer.Operator(statePath, mbdiscid) - - self.stdout.write('MusicBrainz disc id is %s\n' % mbdiscid) - prog.metadata = prog.getMusicBrainz(cueImage.table, mbdiscid, - release=self.options.release_id) - - if not prog.metadata: - print 'Not in MusicBrainz database, skipping' - continue - - # FIXME: this feels like we're poking at internals. - prog.cuePath = arg - prog.result = result.RipResult() - for track in cueImage.table.tracks: - path = cueImage.getRealPath(track.indexes[1].path) - - taglist = prog.getTagList(track.number) - self.debug( - 'possibly retagging %r from cue path %r with taglist %r', - path, arg, taglist) - t = encode.SafeRetagTask(path, taglist) - runner.run(t) - path = os.path.basename(path) - if t.changed: - print 'Retagged %s' % path - else: - print '%s already tagged correctly' % path - print class Verify(logcommand.LogCommand): From 87a424c03a54fe2d359e16d08e9dfbc8d550b5bc Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 16:55:12 -0500 Subject: [PATCH 125/132] Make rip debug encode use the extension from profile by default --- morituri/rip/debug.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/morituri/rip/debug.py b/morituri/rip/debug.py index 678e64b..68a57cc 100644 --- a/morituri/rip/debug.py +++ b/morituri/rip/debug.py @@ -151,6 +151,9 @@ class Encode(logcommand.LogCommand): default=default) def do(self, args): + from morituri.common import encode + profile = encode.ALL_PROFILES[self.options.profile]() + try: fromPath = unicode(args[0]) except IndexError: @@ -160,12 +163,10 @@ class Encode(logcommand.LogCommand): try: toPath = unicode(args[1]) except IndexError: - toPath = fromPath + '.' + self.options.profile + toPath = fromPath + '.' + profile.extension runner = task.SyncRunner() - from morituri.common import encode - profile = encode.ALL_PROFILES[self.options.profile]() self.debug('Encoding %s to %s', fromPath.encode('utf-8'), toPath.encode('utf-8')) From 8a8189fa89bdfb25ca7718f99b638ed85661674e Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 16:55:29 -0500 Subject: [PATCH 126/132] make profiles loggable --- morituri/common/encode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/morituri/common/encode.py b/morituri/common/encode.py index 27cce8f..874dce3 100644 --- a/morituri/common/encode.py +++ b/morituri/common/encode.py @@ -32,7 +32,8 @@ from morituri.common import task as ctask from morituri.extern.task import task, gstreamer -class Profile(object): +class Profile(log.Loggable): + name = None extension = None pipeline = None From aec75861bed2e7374230c18cd9444d55786f91a4 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 16:55:43 -0500 Subject: [PATCH 127/132] Use newer lamemp3enc element This fixes trac issue 45 and adapts pull request 42 on github --- morituri/common/encode.py | 45 ++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/morituri/common/encode.py b/morituri/common/encode.py index 874dce3..7e4e0ee 100644 --- a/morituri/common/encode.py +++ b/morituri/common/encode.py @@ -100,20 +100,45 @@ class WavpackProfile(Profile): lossless = True -class MP3Profile(Profile): +class _LameProfile(Profile): + extension = 'mp3' + lossless = False + + def test(self): + version = cgstreamer.elementFactoryVersion('lamemp3enc') + self.debug('lamemp3enc version: %r', version) + if version: + t = tuple([int(s) for s in version.split('.')]) + if t >= (0, 10, 19): + self.pipeline = self._lamemp3enc_pipeline + return True + + version = cgstreamer.elementFactoryVersion('lame') + self.debug('lame version: %r', version) + if version: + self.pipeline = self._lame_pipeline + return True + + return False + + +class MP3Profile(_LameProfile): name = 'mp3' - extension = 'mp3' - pipeline = 'lame name=tagger quality=0 ! id3v2mux' - lossless = False + + _lame_pipeline = 'lame name=tagger quality=0 ! id3v2mux' + _lamemp3enc_pipeline = \ + 'lamemp3enc name=tagger target=bitrate cbr=true bitrate=320 ! ' \ + 'xingmux ! id3v2mux' -class MP3VBRProfile(Profile): +class MP3VBRProfile(_LameProfile): name = 'mp3vbr' - extension = 'mp3' - pipeline = 'lame name=tagger ' \ - 'vbr-quality=0 vbr=new vbr-mean-bitrate=192 ! ' \ - 'id3v2mux' - lossless = False + + _lame_pipeline = 'lame name=tagger ' \ + 'vbr-quality=0 vbr=new vbr-mean-bitrate=192 ! ' \ + 'id3v2mux' + _lamemp3enc_pipeline = 'lamemp3enc name=tagger quality=0 ' \ + '! xingmux ! id3v2mux' class VorbisProfile(Profile): From 0224cf0080d680d7555c341e709b085223ec22a0 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 22:06:40 -0500 Subject: [PATCH 128/132] README got renamed to README.md; fix build Fixes #52 --- Makefile.am | 2 +- morituri.spec.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 8d8d232..6c09c26 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,7 +5,7 @@ ACLOCAL_AMFLAGS = -I m4 SUBDIRS = morituri bin etc doc m4 misc -EXTRA_DIST = morituri.spec morituri.doap RELEASE README HACKING REVISION +EXTRA_DIST = morituri.spec morituri.doap RELEASE README.md HACKING REVISION SOURCES = $(top_srcdir)/morituri/*.py $(top_srcdir)/morituri/*/*.py diff --git a/morituri.spec.in b/morituri.spec.in index 1b924b9..2efe03b 100644 --- a/morituri.spec.in +++ b/morituri.spec.in @@ -49,7 +49,7 @@ rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) -%doc README morituri.doap NEWS RELEASE ChangeLog +%doc README.md morituri.doap NEWS RELEASE ChangeLog %{_bindir}/rip %{_libdir}/morituri/plugins %{_mandir}/man1/rip.1* From a74fd71fc354d2c6f91d168dc40b13a1dfce58df Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 23 Nov 2013 22:33:31 -0500 Subject: [PATCH 129/132] add a symlink to make autotools happy --- README | 1 + 1 file changed, 1 insertion(+) create mode 120000 README diff --git a/README b/README new file mode 120000 index 0000000..42061c0 --- /dev/null +++ b/README @@ -0,0 +1 @@ +README.md \ No newline at end of file From 274309a8eace4ce9971c2464a48e7555fb71e1fb Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sat, 30 Nov 2013 11:49:29 -0500 Subject: [PATCH 130/132] Fix rip image verify. Fixes github #56. --- morituri/rip/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morituri/rip/image.py b/morituri/rip/image.py index a770f06..2fd668d 100644 --- a/morituri/rip/image.py +++ b/morituri/rip/image.py @@ -161,7 +161,7 @@ Verifies the image from the given .cue files against the AccurateRip database. ''' def do(self, args): - prog = program.Program(self.getRootCommand.config()) + prog = program.Program(self.getRootCommand().config()) runner = task.SyncRunner() cache = accurip.AccuCache() From b461d8a16f051aeda3d5bb43d6654794ac3d331f Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Sun, 1 Dec 2013 22:29:10 -0500 Subject: [PATCH 131/132] Default to 0000 for y when no metadata. Fixes #53 on github. Patch by: tlc --- morituri/common/program.py | 1 + 1 file changed, 1 insertion(+) diff --git a/morituri/common/program.py b/morituri/common/program.py index a65b9c0..d09f1c2 100644 --- a/morituri/common/program.py +++ b/morituri/common/program.py @@ -239,6 +239,7 @@ class Program(log.Loggable): v['C'] = '' # catalog number v['x'] = profile and profile.extension or 'unknown' v['X'] = v['x'].upper() + v['y'] = '0000' v['a'] = v['A'] if i == 0: From 7f1521d50f5db0295f3128718b2af95a7091b5d6 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 16 Dec 2013 23:17:43 -0500 Subject: [PATCH 132/132] Encode directory name when writing to stdout. Fixes #59 on github. Saddening to see though that this was exposed because someone put a unicode dash in an otherwise normal album title. People should stop abusing unicode just because they can. --- morituri/rip/cd.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/morituri/rip/cd.py b/morituri/rip/cd.py index 1689ae0..0183764 100644 --- a/morituri/rip/cd.py +++ b/morituri/rip/cd.py @@ -288,11 +288,12 @@ Install pycdio and run 'rip offset find' to detect your drive's offset. dirname = os.path.dirname(discName) if os.path.exists(dirname): self.stdout.write("Output directory %s already exists\n" % - dirname) + dirname.encode('utf-8')) logs = glob.glob(os.path.join(dirname, '*.log')) if logs: - self.stdout.write("Output directory %s is a finished rip\n" % - dirname) + self.stdout.write( + "Output directory %s is a finished rip\n" % + dirname.encode('utf-8')) if not disambiguate: disambiguate = True continue @@ -301,7 +302,8 @@ Install pycdio and run 'rip offset find' to detect your drive's offset. break else: - self.stdout.write("Creating output directory %s\n" % dirname) + self.stdout.write("Creating output directory %s\n" % + dirname.encode('utf-8')) os.makedirs(dirname) break