From a9d07dd33f00f555ecddfb640c4539467f80857b Mon Sep 17 00:00:00 2001 From: Andrew Weiss Date: Tue, 21 Nov 2017 15:48:26 -0800 Subject: [PATCH 01/13] add support for sectioned source maps --- sourcemap/decoder.py | 59 ++++++++++++++++++++++--- sourcemap/objects.py | 22 ++++++++++ tests/test_objects.py | 100 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 175 insertions(+), 6 deletions(-) diff --git a/sourcemap/decoder.py b/sourcemap/decoder.py index 694ec48..9154b89 100644 --- a/sourcemap/decoder.py +++ b/sourcemap/decoder.py @@ -14,7 +14,7 @@ import sys from functools import partial from .exceptions import SourceMapDecodeError -from .objects import Token, SourceMapIndex +from .objects import Token, SourceMapIndex, SectionedSourceMapIndex try: import simplejson as json except ImportError: @@ -63,8 +63,10 @@ def parse_vlq(self, segment): return values def decode(self, source): - """Decode a source map object into a SourceMapIndex. + """Decode a source map object into a SourceMapIndex or + SectionedSourceMapIndex. + For SourceMapIndex: The index is keyed on (dst_line, dst_column) for lookups, and a per row index is kept to help calculate which Token to retrieve. @@ -102,6 +104,29 @@ def decode(self, source): lte to the bisect_right: 2-1 => row[2-1] => 12 - At this point, we know the token location, (1, 12) - Pull (1, 12) from index => tokens[3] + + For SectionedSourceMapIndex: + The offsets are stored as tuples in sorted order: + [(0, 0), (1, 10), (1, 24), (2, 0), ...] + + For each offset there is a corresponding SourceMapIndex + which operates as described above, except the tokens + are relative to their own section and must have the offset + replied in reverse on the destination row/col when the tokens + are returned. + + To find the token at (1, 20): + - bisect_right to find the closest index (1, 20) + - Supposing that returns index i, we actually want (i - 1) + because the token we want is inside the map before that one + - We then have a SourceMapIndex and we perform the search + for (1 - offset[0], column - offset[1]). [Note this isn't + exactly correct as we have to account for different lines + being searched for and the found offset, so for the column + we use either offset[1] or 0 depending on if line matches + offset[0] or not] + - The token we find we then translate dst_line += offset[0], + and dst_col += offset[1]. """ # According to spec (https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit#heading=h.h7yy76c5il9v) # A SouceMap may be prepended with ")]}'" to cause a Javascript error. @@ -110,6 +135,18 @@ def decode(self, source): source = source.split('\n', 1)[1] smap = json.loads(source) + if smap.get('sections'): + offsets = [] + maps = [] + for section in smap.get('sections'): + offset = section.get('offset') + offsets.append((offset.get('line'), offset.get('column'))) + maps.append(self._decode_map(section.get('map'))) + return SectionedSourceMapIndex(offsets, maps) + else: + return self._decode_map(smap) + + def _decode_map(self, smap): sources = smap['sources'] sourceRoot = smap.get('sourceRoot') names = list(map(text_type, smap['names'])) @@ -197,6 +234,18 @@ def decode(self, source): # Mapping of base64 letter -> integer value. # This weird list is being allocated for faster lookups -B64 = [-1] * 123 -for i, c in enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'): - B64[ord(c)] = i +# B64 = [-1] * 256 +# for i, c in enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'): +# B64[ord(c)] = i +B64 = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, + -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, + -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + 47, 48, 49, 50, 51, -1, -1, -1, -1, -1 - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1]; diff --git a/sourcemap/objects.py b/sourcemap/objects.py index a397dfd..4db27c1 100644 --- a/sourcemap/objects.py +++ b/sourcemap/objects.py @@ -88,3 +88,25 @@ def __len__(self): def __repr__(self): return '' % ', '.join(map(str, self.sources)) + + +class SectionedSourceMapIndex(object): + """The index for a source map which contains sections + containing all the Tokens and precomputed indexes for + searching.""" + + def __init__(self, offsets, maps): + self.offsets = offsets + self.maps = maps + + def lookup(self, line, column): + map_index = bisect_right(self.offsets, (line, column)) - 1 + line_offset, col_offset = self.offsets[map_index] + col_offset = 0 if line != line_offset else col_offset + result = self.maps[map_index].lookup(line - line_offset, column - col_offset) + result.dst_line += line_offset + result.dst_col += col_offset + return result + + def __repr__(self): + return '' % ', '.join(map(str, self.maps)) diff --git a/tests/test_objects.py b/tests/test_objects.py index 852bf1b..fc91219 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -2,7 +2,7 @@ import unittest2 as unittest except ImportError: import unittest -from sourcemap.objects import Token, SourceMapIndex +from sourcemap.objects import Token, SourceMapIndex, SectionedSourceMapIndex class TokenTestCase(unittest.TestCase): @@ -10,6 +10,104 @@ def test_eq(self): assert Token(1, 1, 'lol.js', 1, 1, 'lol') == Token(1, 1, 'lol.js', 1, 1, 'lol') assert Token(99, 1, 'lol.js', 1, 1, 'lol') != Token(1, 1, 'lol.js', 1, 1, 'lol') +class SectionedSourceMapIndexTestCase(unittest.TestCase): + def get_index(self): + offsets = [(0, 0), (1, 14), (2, 28)] + tokens0 = [ + Token(dst_line=0, dst_col=0), + Token(dst_line=0, dst_col=5), + Token(dst_line=1, dst_col=0), + Token(dst_line=1, dst_col=12), + ] + tokens1 = [ + Token(dst_line=0, dst_col=0), + Token(dst_line=0, dst_col=5), + Token(dst_line=1, dst_col=0), + Token(dst_line=1, dst_col=12), + ] + tokens2 = [ + Token(dst_line=0, dst_col=0), + Token(dst_line=0, dst_col=5), + Token(dst_line=1, dst_col=0), + Token(dst_line=1, dst_col=12), + ] + maps = [ + SourceMapIndex({}, tokens0, + [ + [0, 5], + [0, 12], + ], + { + (0, 0): tokens0[0], + (0, 5): tokens0[1], + (1, 0): tokens0[2], + (1, 12): tokens0[3], + }), + SourceMapIndex({}, tokens1, + [ + [0, 5], + [0, 12], + ], + { + (0, 0): tokens1[0], + (0, 5): tokens1[1], + (1, 0): tokens1[2], + (1, 12): tokens1[3], + }), + SourceMapIndex({}, tokens2, + [ + [0, 5], + [0, 12], + ], + { + (0, 0): tokens2[0], + (0, 5): tokens2[1], + (1, 0): tokens2[2], + (1, 12): tokens2[3], + }), + ] + + return SectionedSourceMapIndex(offsets, maps), [tokens0, tokens1, tokens2] + + def test_lookup(self): + index, tokens = self.get_index() + + for i in range(5): + assert index.lookup(0, i) is tokens[0][0] + + for i in range(5, 10): + assert index.lookup(0, i) is tokens[0][1] + + for i in range(12): + assert index.lookup(1, i) is tokens[0][2] + + for i in range(12, 14): + assert index.lookup(1, i) is tokens[0][3] + + for i in range(14, 19): + assert index.lookup(1, i) is tokens[1][0] + + for i in range(19, 25): + assert index.lookup(1, i) is tokens[1][1] + + for i in range(12): + assert index.lookup(2, i) is tokens[1][2] + + for i in range(12, 28): + assert index.lookup(2, i) is tokens[1][3] + + for i in range(28, 33): + assert index.lookup(2, i) is tokens[2][0] + + for i in range(33, 40): + assert index.lookup(2, i) is tokens[2][1] + + for i in range(12): + assert index.lookup(3, i) is tokens[2][2] + + for i in range(12, 14): + assert index.lookup(3, i) is tokens[2][3] + class SourceMapIndexTestCase(unittest.TestCase): def get_index(self): From 6d98645308f0e9368b858119daecfdf916abd0c9 Mon Sep 17 00:00:00 2001 From: Andrew Weiss Date: Wed, 22 Nov 2017 08:05:57 -0800 Subject: [PATCH 02/13] return the source map along with the token from lookup, revert the B64 lookup back to the enumeration --- sourcemap/decoder.py | 18 +++--------------- sourcemap/objects.py | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/sourcemap/decoder.py b/sourcemap/decoder.py index 9154b89..9947506 100644 --- a/sourcemap/decoder.py +++ b/sourcemap/decoder.py @@ -234,18 +234,6 @@ def _decode_map(self, smap): # Mapping of base64 letter -> integer value. # This weird list is being allocated for faster lookups -# B64 = [-1] * 256 -# for i, c in enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'): -# B64[ord(c)] = i -B64 = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, - -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, - -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - 47, 48, 49, 50, 51, -1, -1, -1, -1, -1 - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1]; +B64 = [-1] * 123 +for i, c in enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'): + B64[ord(c)] = i diff --git a/sourcemap/objects.py b/sourcemap/objects.py index 4db27c1..ca6ceb5 100644 --- a/sourcemap/objects.py +++ b/sourcemap/objects.py @@ -61,7 +61,7 @@ def __init__(self, raw, tokens, line_index, index, sources=None): def lookup(self, line, column): try: # Let's hope for a direct match first - return self.index[(line, column)] + return self.index[(line, column)], self except KeyError: pass @@ -75,7 +75,22 @@ def lookup(self, line, column): # We actually want the one less than current column = line_index[i - 1] # Return from the main index, based on the (line, column) tuple - return self.index[(line, column)] + return self.index[(line, column)], self + + def sources_content_map(self): + result = self._source_content_array() + if result: + return dict(result) + else: + return None + + def _source_content_array(self): + sources = self.raw.get('sources') + content = self.raw.get('sourcesContent') + if sources and content: + return zip(sources, content) + else: + return None def __getitem__(self, item): return self.tokens[item] @@ -103,10 +118,22 @@ def lookup(self, line, column): map_index = bisect_right(self.offsets, (line, column)) - 1 line_offset, col_offset = self.offsets[map_index] col_offset = 0 if line != line_offset else col_offset - result = self.maps[map_index].lookup(line - line_offset, column - col_offset) + smap = self.maps[map_index] + result = smap.lookup(line - line_offset, column - col_offset) result.dst_line += line_offset result.dst_col += col_offset - return result + return result, smap + + def sources_content_map(self): + content_maps = [] + for m in self.maps: + source_content_array = m._source_content_array() + if source_content_array: + content_maps.extend(source_content_array) + if len(content_maps): + return dict(content_maps) + else: + return None def __repr__(self): return '' % ', '.join(map(str, self.maps)) From 926b78a8ffbb5dc9a51b6354cd4f0a1ba7d2c973 Mon Sep 17 00:00:00 2001 From: Andrew Weiss Date: Wed, 22 Nov 2017 08:53:36 -0800 Subject: [PATCH 03/13] more changes that are needed for supporting how we were using the internals of this library --- sourcemap/decoder.py | 2 +- sourcemap/objects.py | 60 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/sourcemap/decoder.py b/sourcemap/decoder.py index 9947506..06ba08d 100644 --- a/sourcemap/decoder.py +++ b/sourcemap/decoder.py @@ -142,7 +142,7 @@ def decode(self, source): offset = section.get('offset') offsets.append((offset.get('line'), offset.get('column'))) maps.append(self._decode_map(section.get('map'))) - return SectionedSourceMapIndex(offsets, maps) + return SectionedSourceMapIndex(smap, offsets, maps) else: return self._decode_map(smap) diff --git a/sourcemap/objects.py b/sourcemap/objects.py index ca6ceb5..7d73eae 100644 --- a/sourcemap/objects.py +++ b/sourcemap/objects.py @@ -77,20 +77,29 @@ def lookup(self, line, column): # Return from the main index, based on the (line, column) tuple return self.index[(line, column)], self + def columns_for_line(self, line): + return self.line_index[line] + + def total_number_of_lines(self): + return len(self.line_index) + + def files(self): + f = self.raw.get('file') + return [f] if f else None + def sources_content_map(self): result = self._source_content_array() - if result: - return dict(result) - else: - return None + return dict(result) if result else None + + def raw_sources(self): + return self.raw.get('sources') def _source_content_array(self): sources = self.raw.get('sources') content = self.raw.get('sourcesContent') if sources and content: return zip(sources, content) - else: - return None + return None def __getitem__(self, item): return self.tokens[item] @@ -110,7 +119,8 @@ class SectionedSourceMapIndex(object): containing all the Tokens and precomputed indexes for searching.""" - def __init__(self, offsets, maps): + def __init__(self, raw, offsets, maps): + self.raw = raw self.offsets = offsets self.maps = maps @@ -124,6 +134,33 @@ def lookup(self, line, column): result.dst_col += col_offset return result, smap + # TODO: Test this + def columns_for_line(self, line): + last_map_index = bisect_right(self.offsets, (line+1, 0)) - 1 + first_map_index = bisect_right(self.offsets, (line, 0)) - 1 + columns = [] + for map_index in range(first_map_index, last_map_index + 1): + smap = self.maps[map_index] + line_offset, col_offset = self.offsets[map_index] + smap_line = line - line_offset + smap_cols = smap.columns_for_line(smap_line) + columns.extend([x + col_offset for x in smap_cols]) + return columns + + def total_number_of_lines(self): + result = 0 + for smap in self.maps: + result += smap.total_number_of_lines() + return result + + def files(self): + files = [] + for smap in self.maps: + smap_files = smap.files() + if smap_files: + files.extend(smap_files) + return files if len(files) else None + def sources_content_map(self): content_maps = [] for m in self.maps: @@ -132,8 +169,13 @@ def sources_content_map(self): content_maps.extend(source_content_array) if len(content_maps): return dict(content_maps) - else: - return None + return None + + def raw_sources(self): + sources = [] + for m in self.maps: + sources.extend(m.raw_sources()) + return sources def __repr__(self): return '' % ', '.join(map(str, self.maps)) From 341fb05c0bd66c538e2746af95d647140ac817ad Mon Sep 17 00:00:00 2001 From: Andrew Weiss Date: Wed, 22 Nov 2017 11:53:15 -0800 Subject: [PATCH 04/13] more and better tests --- sourcemap/objects.py | 7 ++-- tests/test_objects.py | 86 +++++++++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/sourcemap/objects.py b/sourcemap/objects.py index 7d73eae..4b9fe0d 100644 --- a/sourcemap/objects.py +++ b/sourcemap/objects.py @@ -129,17 +129,16 @@ def lookup(self, line, column): line_offset, col_offset = self.offsets[map_index] col_offset = 0 if line != line_offset else col_offset smap = self.maps[map_index] - result = smap.lookup(line - line_offset, column - col_offset) + result, _ = smap.lookup(line - line_offset, column - col_offset) result.dst_line += line_offset result.dst_col += col_offset return result, smap - # TODO: Test this def columns_for_line(self, line): - last_map_index = bisect_right(self.offsets, (line+1, 0)) - 1 + last_map_index = bisect_right(self.offsets, (line + 1, 0)) first_map_index = bisect_right(self.offsets, (line, 0)) - 1 columns = [] - for map_index in range(first_map_index, last_map_index + 1): + for map_index in range(first_map_index, last_map_index): smap = self.maps[map_index] line_offset, col_offset = self.offsets[map_index] smap_line = line - line_offset diff --git a/tests/test_objects.py b/tests/test_objects.py index fc91219..723c319 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -32,7 +32,7 @@ def get_index(self): Token(dst_line=1, dst_col=12), ] maps = [ - SourceMapIndex({}, tokens0, + SourceMapIndex({'file': 'foo0.js'}, tokens0, [ [0, 5], [0, 12], @@ -43,7 +43,7 @@ def get_index(self): (1, 0): tokens0[2], (1, 12): tokens0[3], }), - SourceMapIndex({}, tokens1, + SourceMapIndex({'file': 'foo1.js'}, tokens1, [ [0, 5], [0, 12], @@ -54,7 +54,7 @@ def get_index(self): (1, 0): tokens1[2], (1, 12): tokens1[3], }), - SourceMapIndex({}, tokens2, + SourceMapIndex({'file': 'foo2.js'}, tokens2, [ [0, 5], [0, 12], @@ -67,47 +67,81 @@ def get_index(self): }), ] - return SectionedSourceMapIndex(offsets, maps), [tokens0, tokens1, tokens2] + raw = {} + + return SectionedSourceMapIndex(raw, offsets, maps), [tokens0, tokens1, tokens2] def test_lookup(self): index, tokens = self.get_index() for i in range(5): - assert index.lookup(0, i) is tokens[0][0] + assert index.lookup(0, i)[0] is tokens[0][0] for i in range(5, 10): - assert index.lookup(0, i) is tokens[0][1] + assert index.lookup(0, i)[0] is tokens[0][1] for i in range(12): - assert index.lookup(1, i) is tokens[0][2] + assert index.lookup(1, i)[0] is tokens[0][2] for i in range(12, 14): - assert index.lookup(1, i) is tokens[0][3] + assert index.lookup(1, i)[0] is tokens[0][3] for i in range(14, 19): - assert index.lookup(1, i) is tokens[1][0] + assert index.lookup(1, i)[0] is tokens[1][0] for i in range(19, 25): - assert index.lookup(1, i) is tokens[1][1] + assert index.lookup(1, i)[0] is tokens[1][1] for i in range(12): - assert index.lookup(2, i) is tokens[1][2] + assert index.lookup(2, i)[0] is tokens[1][2] for i in range(12, 28): - assert index.lookup(2, i) is tokens[1][3] + assert index.lookup(2, i)[0] is tokens[1][3] for i in range(28, 33): - assert index.lookup(2, i) is tokens[2][0] + assert index.lookup(2, i)[0] is tokens[2][0] for i in range(33, 40): - assert index.lookup(2, i) is tokens[2][1] + assert index.lookup(2, i)[0] is tokens[2][1] for i in range(12): - assert index.lookup(3, i) is tokens[2][2] + assert index.lookup(3, i)[0] is tokens[2][2] for i in range(12, 14): - assert index.lookup(3, i) is tokens[2][3] + assert index.lookup(3, i)[0] is tokens[2][3] + + def test_columns_for_line(self): + index, tokens = self.get_index() + cols = index.columns_for_line(0) + + assert cols[0] is tokens[0][0].dst_col + assert cols[1] is tokens[0][1].dst_col + + cols = index.columns_for_line(1) + + assert len(cols) is 4 + assert cols[0] is tokens[0][2].dst_col + assert cols[1] is tokens[0][3].dst_col + assert cols[2] is tokens[1][0].dst_col + index.offsets[1][1] + assert cols[3] is tokens[1][1].dst_col + index.offsets[1][1] + + cols = index.columns_for_line(2) + assert len(cols) is 4 + assert cols[0] is tokens[1][2].dst_col + index.offsets[1][1] + assert cols[1] is tokens[1][3].dst_col + index.offsets[1][1] + assert cols[2] is tokens[2][0].dst_col + index.offsets[2][1] + assert cols[3] is tokens[2][1].dst_col + index.offsets[2][1] + + def test_lookup_from_columns_for_line(self): + index, tokens = self.get_index() + cols = index.columns_for_line(2) + t, _ = index.lookup(2, cols[2]) + assert t is tokens[2][0] + + def test_files(self): + index, _ = self.get_index() + assert len(index.files()) is 3 class SourceMapIndexTestCase(unittest.TestCase): def get_index(self): @@ -138,16 +172,28 @@ def test_lookup(self): index, tokens = self.get_index() for i in range(5): - assert index.lookup(0, i) is tokens[0] + assert index.lookup(0, i)[0] is tokens[0] for i in range(5, 10): - assert index.lookup(0, i) is tokens[1] + assert index.lookup(0, i)[0] is tokens[1] for i in range(12): - assert index.lookup(1, i) is tokens[2] + assert index.lookup(1, i)[0] is tokens[2] for i in range(12, 20): - assert index.lookup(1, i) is tokens[3] + assert index.lookup(1, i)[0] is tokens[3] + + def test_columns_for_line(self): + index, tokens = self.get_index() + cols = index.columns_for_line(0) + + assert cols[0] is tokens[0].dst_col + assert cols[1] is tokens[1].dst_col + + cols = index.columns_for_line(1) + + assert cols[0] is tokens[2].dst_col + assert cols[1] is tokens[3].dst_col def test_getitem(self): index, tokens = self.get_index() From 50af8c413ef36d425d98a0f6fef2b29a356f4224 Mon Sep 17 00:00:00 2001 From: Andrew Weiss Date: Wed, 22 Nov 2017 12:24:11 -0800 Subject: [PATCH 05/13] I need a version bump to get this to work --- sourcemap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sourcemap/__init__.py b/sourcemap/__init__.py index b40ac40..0d34735 100644 --- a/sourcemap/__init__.py +++ b/sourcemap/__init__.py @@ -8,7 +8,7 @@ from .exceptions import SourceMapDecodeError # NOQA from .decoder import SourceMapDecoder -__version__ = '0.2.1' +__version__ = '0.3.0' def load(fp, cls=None): From 6149a26e731a1dbb040a137625dc6d08de1511b6 Mon Sep 17 00:00:00 2001 From: Israel Gayoso Date: Thu, 10 Jun 2021 01:45:56 +0200 Subject: [PATCH 06/13] Add pull request template --- .github/pull_request_template.md | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..ab666cc --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,34 @@ +## Description of the change + +> Please include a summary of the change and which issues are fixed. +> Please also include relevant motivation and context. + +## Type of change + +- [ ] Bug fix (non-breaking change that fixes an issue) +- [ ] New feature (non-breaking change that adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Maintenance +- [ ] New release + +## Related issues + +> ClubHouse stories and GitHub issues (delete irrelevant) + +- Fix [ch] +- Fix #1 + +## Checklists + +### Development + +- [ ] Lint rules pass locally +- [ ] The code changed/added as part of this pull request has been covered with tests +- [ ] All tests related to the changed code pass in development + +### Code review + +- [ ] This pull request has a descriptive title and information useful to a reviewer. There may be a screenshot or screencast attached +- [ ] "Ready for review" label attached to the PR and reviewers assigned +- [ ] Issue from task tracker has a link to this pull request +- [ ] Changes have been reviewed by at least one other engineer From 40b0756309cdf094f72da4afb8ea5bcbf7d9050e Mon Sep 17 00:00:00 2001 From: Paco Esteban Date: Fri, 2 Dec 2022 11:08:59 +0100 Subject: [PATCH 07/13] add docs to publish to private registry --- README.md | 55 ++++++++++++++++++++++++++++++++++++++ tools/build_and_publish.sh | 17 ++++++++++++ 2 files changed, 72 insertions(+) create mode 100755 tools/build_and_publish.sh diff --git a/README.md b/README.md index 68b1b12..adda1d5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,58 @@ +Rollbar Info +============ + +This is a private clone of https://github.com/requests/requests-oauthlib +so we can build package distributions from it pinned to the version we use. + +As of 2022-12-02, there should be a version on our Python private registry built and ready to use: +https://console.cloud.google.com/artifacts/python/rollbar-prod/us-central1/python-private-registry/sourcemap?project=rollbar-prod + +This corresponds to the commit pinned as a dependency for mox at the time of writing: `05735efbd5c8cdcaff0c2ca3b341dafc3d1dbadb` +And it's marked as `0.3.0+05735ef`. + +It has been build following this procedure, which can be replicated if we ever need to switch to a more recent version or push a patch to it. +You'll need a service account that has the `roles/artifactregistry.writer`. This is needed for Python2, as we cannot use the keyring auth provider. +With that, you can see the config needed for twine. +Execute this command: + +``` +gcloud artifacts print-settings python \ + --repository=python-private-registry \ + --project=rollbar-prod \ + --location=us-central1 \ + --json-key=/path/to/my-sa-key.json +``` + +That will produce something like this: +``` +# Insert the following snippet into your .pypirc + +[distutils] +index-servers = + python-private-registry + +[python-private-registry] +repository: https://us-central1-python.pkg.dev/rollbar-prod/python-private-registry/ +username: _json_key_base64 +password: ... + +# Insert the following snippet into your pip.conf + +[global] +extra-index-url = https://_json_key_base64:...@us-central1-python.pkg.dev/rollbar-prod/python-private-registry/simple/ +``` + +We're only interested on the username and password of the registry. Copy those somewhere. + +Now: + +- checkout the repo to the commit or tag you need +- modify `sourcemap/__init__.py` to reflect the new version. Always + append the `+HASH` to the version in order not to conflict with upstream. +- launch a shell into a docker image with Python 2.7 support (password is the base64 string from before): + `docker run -ti --rm -v $(pwd):/app -e TWINE_USERNAME=_json_key_base64 -e TWINE_PASSWORD=<...> cimg/python:2.7 /bin/bash` +- launch the included `./tools/build_and_publish.sh` script + # SourceMap [![Build Status](https://travis-ci.org/mattrobenolt/python-sourcemap.png?branch=master)](https://travis-ci.org/mattrobenolt/python-sourcemap) Parse JavaScript source maps. diff --git a/tools/build_and_publish.sh b/tools/build_and_publish.sh new file mode 100755 index 0000000..68f6131 --- /dev/null +++ b/tools/build_and_publish.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -eu + +cd /app || exit 1 + +# install the twine tool to upload the python distributions +pip install twine + +# build the distributions: +python setup.py sdist bdist_wheel + +# upload to registry +twine upload \ + --repository-url https://us-central1-python.pkg.dev/rollbar-prod/python-private-registry/ \ + --verbose \ + dist/* From 578d9da18bb539e2f00f68a9d75bb043b2fdca7e Mon Sep 17 00:00:00 2001 From: Paco Esteban Date: Fri, 2 Dec 2022 11:19:17 +0100 Subject: [PATCH 08/13] fix readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index adda1d5..caaa1fa 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Rollbar Info ============ -This is a private clone of https://github.com/requests/requests-oauthlib +This is a private clone of https://github.com/mattrobenolt/python-sourcemap so we can build package distributions from it pinned to the version we use. As of 2022-12-02, there should be a version on our Python private registry built and ready to use: From ff0e71460b8ed29c1abc233610215025ece1c52a Mon Sep 17 00:00:00 2001 From: Paco Esteban Date: Tue, 17 Jan 2023 10:57:04 +0100 Subject: [PATCH 09/13] change docs to publish to the devpi registry --- README.md | 37 +++++++------------------------------ tools/build_and_publish.sh | 2 +- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index caaa1fa..5734588 100644 --- a/README.md +++ b/README.md @@ -5,52 +5,29 @@ This is a private clone of https://github.com/mattrobenolt/python-sourcemap so we can build package distributions from it pinned to the version we use. As of 2022-12-02, there should be a version on our Python private registry built and ready to use: -https://console.cloud.google.com/artifacts/python/rollbar-prod/us-central1/python-private-registry/sourcemap?project=rollbar-prod +https://pypi.rollbar.tools This corresponds to the commit pinned as a dependency for mox at the time of writing: `05735efbd5c8cdcaff0c2ca3b341dafc3d1dbadb` And it's marked as `0.3.0+05735ef`. It has been build following this procedure, which can be replicated if we ever need to switch to a more recent version or push a patch to it. -You'll need a service account that has the `roles/artifactregistry.writer`. This is needed for Python2, as we cannot use the keyring auth provider. -With that, you can see the config needed for twine. -Execute this command: +You'll need the username and password of the registry's write user to publish. +The credentials are on LastPass, under the `devpi rollbar user` entry. -``` -gcloud artifacts print-settings python \ - --repository=python-private-registry \ - --project=rollbar-prod \ - --location=us-central1 \ - --json-key=/path/to/my-sa-key.json -``` +Export these environment variables to use with twine later: -That will produce something like this: ``` -# Insert the following snippet into your .pypirc - -[distutils] -index-servers = - python-private-registry - -[python-private-registry] -repository: https://us-central1-python.pkg.dev/rollbar-prod/python-private-registry/ -username: _json_key_base64 -password: ... - -# Insert the following snippet into your pip.conf - -[global] -extra-index-url = https://_json_key_base64:...@us-central1-python.pkg.dev/rollbar-prod/python-private-registry/simple/ +export TWINE_USERNAME=rollbar +export TWINE_PASSWORD=...... # get it from LastPass ``` -We're only interested on the username and password of the registry. Copy those somewhere. - Now: - checkout the repo to the commit or tag you need - modify `sourcemap/__init__.py` to reflect the new version. Always append the `+HASH` to the version in order not to conflict with upstream. - launch a shell into a docker image with Python 2.7 support (password is the base64 string from before): - `docker run -ti --rm -v $(pwd):/app -e TWINE_USERNAME=_json_key_base64 -e TWINE_PASSWORD=<...> cimg/python:2.7 /bin/bash` + `docker run -ti --rm -v $(pwd):/app -e TWINE_USERNAME=$TWINE_USERNAME -e TWINE_PASSWORD=$TWINE_PASSWORD cimg/python:2.7 /bin/bash` - launch the included `./tools/build_and_publish.sh` script # SourceMap [![Build Status](https://travis-ci.org/mattrobenolt/python-sourcemap.png?branch=master)](https://travis-ci.org/mattrobenolt/python-sourcemap) diff --git a/tools/build_and_publish.sh b/tools/build_and_publish.sh index 68f6131..73054ec 100755 --- a/tools/build_and_publish.sh +++ b/tools/build_and_publish.sh @@ -12,6 +12,6 @@ python setup.py sdist bdist_wheel # upload to registry twine upload \ - --repository-url https://us-central1-python.pkg.dev/rollbar-prod/python-private-registry/ \ + --repository-url https://pypi.rollbar.tools/rollbar/rollbar/ \ --verbose \ dist/* From bfca04fc925dae9f7bb6761e881fd4f14b50d690 Mon Sep 17 00:00:00 2001 From: Matias Pequeno Date: Wed, 25 Sep 2024 15:42:46 -0300 Subject: [PATCH 10/13] Fix usage of message attr in Exception object (#5) --- sourcemap/decoder.py | 3 ++- tests/test_integration.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sourcemap/decoder.py b/sourcemap/decoder.py index 06ba08d..e521e69 100644 --- a/sourcemap/decoder.py +++ b/sourcemap/decoder.py @@ -215,9 +215,10 @@ def _decode_map(self, smap): assert src_line >= 0, ('src_line', src_line) assert src_col >= 0, ('src_col', src_col) except AssertionError as e: + error_info = e.args[0] raise SourceMapDecodeError( "Segment %s has negative %s (%d), in file %s" - % (segment, e.message[0], e.message[1], src) + % (segment, error_info[0], error_info[1], src) ) token = Token(dst_line, dst_col, src, src_line, src_col, name) diff --git a/tests/test_integration.py b/tests/test_integration.py index 8755076..74f72ab 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -65,3 +65,13 @@ def test_unicode_names(self): # This shouldn't blow up sourcemap.loads(min_map) + + def test_invalid_map(self): + with self.assertRaises( + sourcemap.SourceMapDecodeError, + msg='Segment LCnBD has negative dst_col (-5), in file test-invalid2.js' + ): + sourcemap.loads( + '{"version":3,"lineCount":1,"mappings":"LCnBD;",' + '"sources":["test-invalid.js","test-invalid2.js"],"names":[]}' + ) From 7cb7731f6a3a9c418ab54f12e1719f724447d307 Mon Sep 17 00:00:00 2001 From: Dan Milgram Date: Thu, 6 Mar 2025 16:16:23 -0300 Subject: [PATCH 11/13] raise custom error --- sourcemap/decoder.py | 5 ++++- sourcemap/exceptions.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sourcemap/decoder.py b/sourcemap/decoder.py index e521e69..36e272f 100644 --- a/sourcemap/decoder.py +++ b/sourcemap/decoder.py @@ -13,7 +13,7 @@ import os import sys from functools import partial -from .exceptions import SourceMapDecodeError +from .exceptions import SourceMapDecodeError, SourceMapInvalidError from .objects import Token, SourceMapIndex, SectionedSourceMapIndex try: import simplejson as json @@ -148,6 +148,9 @@ def decode(self, source): def _decode_map(self, smap): sources = smap['sources'] + if not all(isinstance(item, str) for item in sources): + raise SourceMapInvalidError("Sources must be a list of strings") + sourceRoot = smap.get('sourceRoot') names = list(map(text_type, smap['names'])) mappings = smap['mappings'] diff --git a/sourcemap/exceptions.py b/sourcemap/exceptions.py index 9e0350c..7a4fd53 100644 --- a/sourcemap/exceptions.py +++ b/sourcemap/exceptions.py @@ -8,3 +8,7 @@ class SourceMapDecodeError(ValueError): "lol sourcemap error" pass + +class SourceMapInvalidError(TypeError): + "invalid sourcemap due to a type error" + pass From c31fbdf204a79676cb90332592f5e5ba9f2199fd Mon Sep 17 00:00:00 2001 From: Dan Milgram Date: Thu, 6 Mar 2025 17:45:05 -0300 Subject: [PATCH 12/13] use better type name + add test --- sourcemap/decoder.py | 4 ++-- sourcemap/exceptions.py | 2 +- tests/test_integration.py | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/sourcemap/decoder.py b/sourcemap/decoder.py index 36e272f..331cd53 100644 --- a/sourcemap/decoder.py +++ b/sourcemap/decoder.py @@ -13,7 +13,7 @@ import os import sys from functools import partial -from .exceptions import SourceMapDecodeError, SourceMapInvalidError +from .exceptions import SourceMapDecodeError, SourceMapTypeError from .objects import Token, SourceMapIndex, SectionedSourceMapIndex try: import simplejson as json @@ -149,7 +149,7 @@ def decode(self, source): def _decode_map(self, smap): sources = smap['sources'] if not all(isinstance(item, str) for item in sources): - raise SourceMapInvalidError("Sources must be a list of strings") + raise SourceMapTypeError("Sources must be a list of strings") sourceRoot = smap.get('sourceRoot') names = list(map(text_type, smap['names'])) diff --git a/sourcemap/exceptions.py b/sourcemap/exceptions.py index 7a4fd53..4c75486 100644 --- a/sourcemap/exceptions.py +++ b/sourcemap/exceptions.py @@ -9,6 +9,6 @@ class SourceMapDecodeError(ValueError): "lol sourcemap error" pass -class SourceMapInvalidError(TypeError): +class SourceMapTypeError(TypeError): "invalid sourcemap due to a type error" pass diff --git a/tests/test_integration.py b/tests/test_integration.py index 74f72ab..3709446 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -75,3 +75,12 @@ def test_invalid_map(self): '{"version":3,"lineCount":1,"mappings":"LCnBD;",' '"sources":["test-invalid.js","test-invalid2.js"],"names":[]}' ) + + def test_invalid_map_type_error(self): + with self.assertRaises( + sourcemap.exceptions.SourceMapTypeError, + msg='Sources must be a list of strings' + ): + sourcemap.loads( + '{"version":3,"sources":["1", "2", 3],"names":["x","alert"],"mappings":"AAAA,GAAIA,GAAI,EACR,IAAIA,GAAK,EAAG,CACVC,MAAM"}' + ) From 5eed3a45c902e47a6221a43dc2042bdb8f117a00 Mon Sep 17 00:00:00 2001 From: Dan Milgram Date: Fri, 7 Mar 2025 10:12:03 -0300 Subject: [PATCH 13/13] version 0.3.1 --- sourcemap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sourcemap/__init__.py b/sourcemap/__init__.py index 0d34735..208fd2e 100644 --- a/sourcemap/__init__.py +++ b/sourcemap/__init__.py @@ -8,7 +8,7 @@ from .exceptions import SourceMapDecodeError # NOQA from .decoder import SourceMapDecoder -__version__ = '0.3.0' +__version__ = '0.3.1' def load(fp, cls=None):