from __future__ import print_function """ Defines GeobaseReader, which reads geobase (a geography database from Ray Mooney's group at UT Austin) from a Prolog file and parses it into a set of triples, each consisting of: - the relation name, one of 17 described below - the source (or subject), an entity id such as '/state/nevada' - the destination (or object), an entity id or a numeric value An entity id resembles a Unix pathname. The first segment indicates the entity's type, while the rest is an identifier based on the entity's ordinary name. There are eight types, or unary relations, listed here by number of instances: 386 city 102 place 51 state 50 mountain 46 river 40 road 22 lake 1 country 698 TOTAL Note that 'place' is the type used for places identified by geobase as the highest or lowest point in a state. Highest points are typically mountains; lowest points are often rivers or lakes. Consequently, entity ids may not be unique identifiers of entities. For example, we have both '/mountain/mckinley' and '/place/mount_mckinley', and '/lake/erie' and '/place/lake_erie'. There are 14 ordinary binary relations, which contain the primary geographical content of geobase. 742 contains # state to city or mountain, or country to state '/state/alaska' => '/city/anchorage_ak' 438 population # city, state, or country to integer '/state/nevada' => 800500 436 borders # state to state, with symmetry '/state/nevada' <=> '/state/utah' 367 traverses # river, road, or lake to state '/road/15' => '/state/nevada' 152 height # mountain to height in meters, or high-low place to elevation in meters 74 area # state, country, or lake to area in square meters '/state/nevada' => 286193686192.128 51 abbreviation # state to two-letter abbrevation '/state/nevada' => 'nv' 51 capital # state to capital city '/state/oregon' => '/city/salem_or' 51 highest_elevation # state to highest elevation in meters '/state/illinois' => 376 51 highest_point # state to place '/state/illinois' => '/place/charles_mound' 51 lowest_elevation # state to lowest elevation in meters '/state/illinois' => 85 51 lowest_point # state to place '/state/illinois' => '/place/mississippi_river' 51 state_number # state to integer '/state/nevada' => 36 46 length # river to length in meters '/river/snake' => 1670000 Note that the source file geobase.pl is not consistent about units, but here we convert everything to SI (metric) units. There is also one special binary relation, which records the mapping between names and ids. 698 name # from entity id to ordinary name: '/city/reno_nv' => 'reno' The Prolog file defining geobase can be obtained from ftp://ftp.cs.utexas.edu/pub/mooney/nl-ilp-data/geosystem/geobase Here we assume it can be read from the local directory as geobase.pl. """ __author__ = "Bill MacCartney" __copyright__ = "Copyright 2015, Bill MacCartney" __credits__ = [] __license__ = "GNU General Public License, version 2.0" __version__ = "0.9" __maintainer__ = "Bill MacCartney" __email__ = "See the author's website" import re import sys from six.moves import urllib valid_line = re.compile(r'^[a-z]+\((.*)\)\.$') def filter_by_prefix(prefix, lines): return [line for line in lines if line.startswith(prefix)] def strip_brackets(string): if string.startswith('['): string = string[1:] if string.endswith(']'): string = string[:-1] return string def strip_quotes(string): if string.startswith("'") and string.endswith("'"): return string[1:-1] else: return string def extract_fields(line): fields = valid_line.match(line).group(1).split(',') fields = [strip_quotes(strip_brackets(field.strip())) for field in fields] fields = [field for field in fields if field != ''] return fields def make_state_id(state_name): return "/state/%s" % state_name.replace(" ", "_") def make_city_id(city_name, state_abbrev): return "/city/%s_%s" % (city_name.replace(" ", "_"), state_abbrev) def make_river_id(river_name): return "/river/%s" % river_name.replace(" ", "_") def make_place_id(place_name): return "/place/%s" % place_name.replace(" ", "_") def make_mountain_id(mountain_name): return "/mountain/%s" % mountain_name.replace(" ", "_") def make_road_id(road_name): return "/road/%s" % road_name.replace(" ", "_") def make_lake_id(lake_name): return "/lake/%s" % lake_name.replace(" ", "_") def make_country_id(country_name): return "/country/%s" % country_name.replace(" ", "_") class GeobaseReader: def __init__(self): self.tuples = set() self.prolog_file = '/tmp/geobase.pl' self.ensure_prolog_file() lines = self.read_lines() self.parse(lines) self.transitive_closure('contains') def ensure_prolog_file(self): try: f = open(self.prolog_file, 'rU') except IOError as e: print('No local cache for Geobase Prolog file', file=sys.stderr) self.download_prolog_file() def download_prolog_file(self): try: from_url = 'ftp://ftp.cs.utexas.edu/pub/mooney/nl-ilp-data/geosystem/geobase' print('Downloading from %s' % from_url, file=sys.stderr) opener = urllib.request.URLopener() opener.retrieve(from_url, self.prolog_file) print('Download successful', file=sys.stderr) except IOError as e: print('Download failed!', file=sys.stderr) raise e def read_lines(self): lines = [] try: f = open(self.prolog_file, 'rU') for line in f.readlines(): if valid_line.match(line): lines.append(line[:-1]) f.close() except IOError as e: print('No local cache for Geobase Prolog file', file=sys.stderr) raise e return lines def parse(self, lines): self.parse_state(lines) self.parse_city(lines) self.parse_river(lines) self.parse_border(lines) self.parse_highlow(lines) self.parse_mountain(lines) self.parse_road(lines) self.parse_lake(lines) self.parse_country(lines) def parse_state(self, lines): lines = filter_by_prefix('state', lines) for line in lines: # state(name, abbreviation, capital, population, area, state_number, city1, city2, city3, city4) fields = extract_fields(line) state_name = fields[0] state_abbrev = fields[1] state_id = make_state_id(state_name) capital_id = make_city_id(fields[2], state_abbrev) population = int(float(fields[3])) # convert from square miles to square meters area = int(float(fields[4])) * 1609.344 * 1609.344 state_number = int(fields[5]) city1_id = make_city_id(fields[6], state_abbrev) city2_id = make_city_id(fields[7], state_abbrev) city3_id = make_city_id(fields[8], state_abbrev) city4_id = make_city_id(fields[9], state_abbrev) self.add_unary('state', state_id) self.add_binary('name', state_id, state_name) self.add_binary('abbreviation', state_id, state_abbrev) self.add_binary('capital', state_id, capital_id) self.add_binary('contains', state_id, capital_id) self.add_binary('population', state_id, population) self.add_binary('area', state_id, area) self.add_binary('state_number', state_id, state_number) self.add_binary('contains', state_id, city1_id) self.add_binary('contains', state_id, city2_id) self.add_binary('contains', state_id, city3_id) self.add_binary('contains', state_id, city4_id) self.add_binary('contains', '/country/usa', state_id) print('GeobaseReader read %d state rows.' % len(lines)) def parse_city(self, lines): lines = filter_by_prefix('city', lines) for line in lines: # city(state, state_abbreviation, name, population) fields = extract_fields(line) state_name = fields[0] state_id = make_state_id(state_name) state_abbrev = fields[1] city_name = fields[2] city_id = make_city_id(city_name, state_abbrev) population = int(fields[3]) self.add_unary('city', city_id) self.add_binary('name', city_id, city_name) self.add_binary('contains', state_id, city_id) self.add_binary('population', city_id, population) print('GeobaseReader read %d city rows.' % len(lines)) def parse_river(self, lines): lines = filter_by_prefix('river', lines) for line in lines: # river(name, length, [states through which it flows]) fields = extract_fields(line) river_name = fields[0] river_id = make_river_id(river_name) # convert kilometers to meters length = int(fields[1]) * 1000 traversed_state_names = fields[2:] self.add_unary('river', river_id) self.add_binary('name', river_id, river_name) self.add_binary('length', river_id, length) for state_name in traversed_state_names: self.add_binary('traverses', river_id, make_state_id(state_name)) print('GeobaseReader read %d river rows.' % len(lines)) def parse_border(self, lines): lines = filter_by_prefix('border', lines) for line in lines: # border(state, state_abbreviation, [states that border it]) fields = extract_fields(line) state_name = fields[0] state_id = make_state_id(state_name) bordered_state_names = fields[2:] for bordered_state_name in bordered_state_names: bordered_state_id = make_state_id(bordered_state_name) self.add_binary('borders', state_id, bordered_state_id) self.add_binary('borders', bordered_state_id, state_id) print('GeobaseReader read %d border rows.' % len(lines)) def parse_highlow(self, lines): lines = filter_by_prefix('highlow', lines) for line in lines: # highlow(state, state_abbreviation, highest_point, highest_elevation, lowest_point, lowest_elevation) fields = extract_fields(line) state_name = fields[0] state_id = make_state_id(state_name) highest_point_name = fields[2] # TODO: consider whether it should be a mountain id highest_point_id = make_place_id(highest_point_name) highest_elevation = int(fields[3]) lowest_point_name = fields[4] lowest_point_id = make_place_id(lowest_point_name) lowest_elevation = int(fields[5]) # TODO: consider whether the type should be mountain, not place self.add_unary('place', highest_point_id) self.add_binary('name', highest_point_id, highest_point_name) self.add_unary('place', lowest_point_id) self.add_binary('name', lowest_point_id, lowest_point_name) self.add_binary('highest_point', state_id, highest_point_id) self.add_binary('highest_elevation', state_id, highest_elevation) self.add_binary('height', highest_point_id, highest_elevation) self.add_binary('lowest_point', state_id, lowest_point_id) self.add_binary('lowest_elevation', state_id, lowest_elevation) self.add_binary('height', lowest_point_id, lowest_elevation) print('GeobaseReader read %d highlow rows.' % len(lines)) def parse_mountain(self, lines): lines = filter_by_prefix('mountain', lines) for line in lines: # mountain(state, state_abbreviation, name, height) fields = extract_fields(line) state_name = fields[0] state_id = make_state_id(state_name) mountain_name = fields[2] mountain_id = make_mountain_id(mountain_name) height = int(fields[3]) self.add_unary('mountain', mountain_id) self.add_binary('name', mountain_id, mountain_name) self.add_binary('contains', state_id, mountain_id) self.add_binary('height', mountain_id, height) print('GeobaseReader read %d mountain rows.' % len(lines)) def parse_road(self, lines): lines = filter_by_prefix('road', lines) for line in lines: # road(number, [states it passes through]) fields = extract_fields(line) road_name = fields[0] road_id = make_road_id(road_name) traversed_state_names = fields[1:] self.add_unary('road', road_id) self.add_binary('name', road_id, road_name) for traversed_state_name in traversed_state_names: self.add_binary('traverses', road_id, make_state_id(traversed_state_name)) print('GeobaseReader read %d road rows.' % len(lines)) def parse_lake(self, lines): lines = filter_by_prefix('lake', lines) for line in lines: # lake(name, area, [states it is in]) fields = extract_fields(line) lake_name = fields[0] lake_id = make_lake_id(fields[0]) # convert from square kilometers to square meters area = int(fields[1]) * 1e6 traversed_state_names = fields[2:] self.add_unary('lake', lake_id) self.add_binary('name', lake_id, lake_name) self.add_binary('area', lake_id, area) for traversed_state_name in traversed_state_names: # 'traverses' may sound odd here, but, logically, it's the same relation. self.add_binary('traverses', lake_id, make_state_id(traversed_state_name)) print('GeobaseReader read %d lake rows.' % len(lines)) def parse_country(self, lines): lines = filter_by_prefix('country', lines) for line in lines: # country(name, population, area) fields = extract_fields(line) country_name = fields[0] country_id = make_country_id(country_name) population = int(fields[1]) # convert from square kilometers to square meters area = int(fields[2]) * 1e6 self.add_unary('country', country_id) self.add_binary('name', country_id, country_name) self.add_binary('population', country_id, population) self.add_binary('area', country_id, area) print('GeobaseReader read %d country row.' % len(lines)) def add_unary(self, rel, elt): self.tuples.add((rel, elt)) # print 'add_unary(%s, %s)' % (rel, elt) def add_binary(self, rel, src, dst): self.tuples.add((rel, src, dst)) # print 'add_binary(%s, %s, %s)' % (rel, src, dst) def transitive_closure(self, rel): edges = [edge for edge in self.tuples if edge[0] == rel] for edge_1 in edges: for edge_2 in edges: if edge_1[2] == edge_2[1]: edges.append((rel, edge_1[1], edge_2[2])) before_size = len(self.tuples) for edge in edges: self.tuples.add(edge) print('GeobaseReader computed transitive closure of \'%s\', adding %d edges' % ( rel, len(self.tuples) - before_size)) if __name__ == '__main__': geobase = GeobaseReader()