From 6bcf4ac235f27bc6a4c0e3f69aac134fda8e331b Mon Sep 17 00:00:00 2001 From: glyn Date: Wed, 11 Nov 2015 14:19:18 +0000 Subject: [PATCH] Mark escaped strings as so --- os_postgresql_import.plpgsql | 10 +++++----- paf_postgresql_import.plpgsql | 4 ++-- paf_postgresql_import_postgis.plpgsql | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/os_postgresql_import.plpgsql b/os_postgresql_import.plpgsql index 8b7dd19..cb15a30 100644 --- a/os_postgresql_import.plpgsql +++ b/os_postgresql_import.plpgsql @@ -101,8 +101,8 @@ BEGIN INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('S92000003', 'CNY', 'Scotland'); INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('E05008322', 'DIW', 'Bryher Ward'); INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('E05008323', 'DIW', 'St. Agnes Ward'); - INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('E05008324', 'DIW', 'St. Martin\'s Ward'); - INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('E05008325', 'DIW', 'St. Mary\'s Ward'); + INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('E05008324', 'DIW', E'St. Martin\'s Ward'); + INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('E05008325', 'DIW', E'St. Mary\'s Ward'); INSERT INTO public.pc_opendata_areas (code, type, description) VALUES ('E05008326', 'DIW', 'Tresco Ward'); -- Finally our postcode data @@ -212,8 +212,8 @@ BEGIN INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('S92000003', 'CNY', 'Scotland'); INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('E05008322', 'DIW', 'Bryher Ward'); INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('E05008323', 'DIW', 'St. Agnes Ward'); - INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('E05008324', 'DIW', 'St. Martin\'s Ward'); - INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('E05008325', 'DIW', 'St. Mary\'s Ward'); + INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('E05008324', 'DIW', E'St. Martin\'s Ward'); + INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('E05008325', 'DIW', E'St. Mary\'s Ward'); INSERT INTO tmp_pc_opendata_areas (code, type, description) VALUES ('E05008326', 'DIW', 'Tresco Ward'); -- Finally our postcode data @@ -243,4 +243,4 @@ BEGIN END; $BODY$ -LANGUAGE 'plpgsql' VOLATILE; \ No newline at end of file +LANGUAGE 'plpgsql' VOLATILE; diff --git a/paf_postgresql_import.plpgsql b/paf_postgresql_import.plpgsql index 7955083..13f47cb 100644 --- a/paf_postgresql_import.plpgsql +++ b/paf_postgresql_import.plpgsql @@ -334,7 +334,7 @@ BEGIN COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_reference_east IS 'BT postcodes reflect the Irish Grid public (different origin and scale); subtract 17000 to approximate onto OS National Grid. For latlong do not approximate but do proper conversion'; COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_reference_north IS 'BT postcodes reflect the Irish Grid public (different origin and scale); add 13000 to approximate onto OS National Grid. For latlong do not approximate but do proper conversion'; COMMENT ON COLUMN public.pc_paf_postzon_100m.user_type IS '0 Small User 1 Large User'; - COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_status IS '0 Status not supplied by OS \n + COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_status IS E'0 Status not supplied by OS \n 1 Within the building of the matched address closest to the Postcode mean. \n 2 Co-ordinates allocated by GROS during Postcode boundary creation to the building nearest the centre of the populated part of the Postcode (Scotland only) \n 3 Approximate to within 50m of true position \n @@ -1032,7 +1032,7 @@ BEGIN -- If I did have a postzon changes file, it'd most likely be best to import the data directly and use postgis to do our conversion -- as the data is selected out of the data_stage table (rather than using the convert_paf.pl script). We could get those fields with -- something like: - -- CASE WHEN substring(northing,1,1) ~ '[\x4f-\x5a]' THEN '1' || translate(northing, 'POUTZY', '221100') ELSE northing END || '0' AS f_notrhing, + -- CASE WHEN substring(northing,1,1) ~ E'[\x4f-\x5a]' THEN '1' || translate(northing, 'POUTZY', '221100') ELSE northing END || '0' AS f_notrhing, -- ST_x(ST_transform(ST_GeomFromText('POINT('||easting||' '||f_northing||')',CASE WHEN postcode LIKE 'BT%' THEN 29903 ELSE 27700 END),4326)) AS longitude, -- ST_y(ST_transform(ST_GeomFromText('POINT('||easting||' '||f_northing||')',CASE WHEN postcode LIKE 'BT%' THEN 29903 ELSE 27700 END),4326)) AS latitude; diff --git a/paf_postgresql_import_postgis.plpgsql b/paf_postgresql_import_postgis.plpgsql index 36c764c..ffc4db8 100644 --- a/paf_postgresql_import_postgis.plpgsql +++ b/paf_postgresql_import_postgis.plpgsql @@ -309,7 +309,7 @@ BEGIN nullif(trim(substring(data,5,3)),'') AS inward_code, to_date(substring(data,8,6), 'YYYYMM') AS introduction_date, (nullif(trim(substring(data,14,5)),'') || '0')::integer AS grid_reference_east, - (CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ '[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')::integer AS grid_reference_north, + (CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ E'[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')::integer AS grid_reference_north, nullif(trim(substring(data,24,9)),'') AS country_code, nullif(trim(substring(data,33,9)),'') AS area_code_county, nullif(trim(substring(data,42,9)),'') AS area_code_district, @@ -318,8 +318,8 @@ BEGIN nullif(trim(substring(data,69,9)),'') AS nhs_code, nullif(trim(substring(data,78,1)),'')::smallint AS user_type, nullif(trim(substring(data,79,1)),'')::smallint AS grid_status, - ST_y(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ '[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS latitude, - ST_x(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ '[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS longitude + ST_y(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ E'[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS latitude, + ST_x(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ E'[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS longitude FROM data_stage; GET DIAGNOSTICS v_processed = ROW_COUNT; @@ -334,7 +334,7 @@ BEGIN COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_reference_east IS 'BT postcodes reflect the Irish Grid public (different origin and scale); subtract 17000 to approximate onto OS National Grid. For latlong do not approximate but do proper conversion'; COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_reference_north IS 'BT postcodes reflect the Irish Grid public (different origin and scale); add 13000 to approximate onto OS National Grid. For latlong do not approximate but do proper conversion'; COMMENT ON COLUMN public.pc_paf_postzon_100m.user_type IS '0 Small User 1 Large User'; - COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_status IS '0 Status not supplied by OS \n + COMMENT ON COLUMN public.pc_paf_postzon_100m.grid_status IS E'0 Status not supplied by OS \n 1 Within the building of the matched address closest to the Postcode mean. \n 2 Co-ordinates allocated by GROS during Postcode boundary creation to the building nearest the centre of the populated part of the Postcode (Scotland only) \n 3 Approximate to within 50m of true position \n @@ -1076,7 +1076,7 @@ BEGIN nullif(trim(substring(data,5,3)),'') AS inward_code, to_date(substring(data,8,6), 'YYYYMM') AS introduction_date, (nullif(trim(substring(data,14,5)),'') || '0')::integer AS grid_reference_east, - (CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ '[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')::integer AS grid_reference_north, + (CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ E'[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')::integer AS grid_reference_north, nullif(trim(substring(data,24,9)),'') AS country_code, nullif(trim(substring(data,33,9)),'') AS area_code_county, nullif(trim(substring(data,42,9)),'') AS area_code_district, @@ -1085,8 +1085,8 @@ BEGIN nullif(trim(substring(data,69,9)),'') AS nhs_code, nullif(trim(substring(data,78,1)),'')::smallint AS user_type, nullif(trim(substring(data,79,1)),'')::smallint AS grid_status, - ST_y(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ '[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS latitude, - ST_x(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ '[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS longitude + ST_y(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ E'[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS latitude, + ST_x(ST_transform(ST_GeomFromText('POINT('||(nullif(trim(substring(data,14,5)),'') || '0')||' '||(CASE WHEN substring(nullif(trim(substring(data,19,5)),''),1,1) ~ E'[\x4f-\x5a]' THEN '1' || translate(nullif(trim(substring(data,19,5)),''), 'POUTZY', '221100') ELSE nullif(trim(substring(data,19,5)),'') END || '0')||')',CASE WHEN nullif(trim(substring(data,1,4)),'') LIKE 'BT%' THEN 29903 ELSE 27700 END),4326))::numeric(8,6) AS longitude FROM data_stage; RAISE NOTICE '%: Done staging postzon with lat long data, updating', clock_timestamp(); -- 2.39.5