Newer
Older
osmCoverage / src / osm / jp / api / Osmdb.java
@hayashi hayashi on 5 Jul 2018 21 KB update
  1. package osm.jp.api;
  2.  
  3. import java.io.*;
  4. import java.sql.Connection;
  5. import java.sql.PreparedStatement;
  6. import java.sql.ResultSet;
  7. import java.sql.SQLException;
  8. import java.sql.SQLIntegrityConstraintViolationException;
  9. import java.sql.SQLSyntaxErrorException;
  10. import javax.json.Json;
  11. import javax.json.JsonArray;
  12. import javax.json.JsonBuilderFactory;
  13. import javax.json.JsonObject;
  14. import javax.json.JsonObjectBuilder;
  15. import javax.json.JsonValue;
  16. import jp.co.areaweb.tools.csv.CsvRecord;
  17. import jp.co.areaweb.tools.database.DatabaseTool;
  18. import org.hsqldb.HsqlException;
  19. import org.w3c.dom.NamedNodeMap;
  20. import org.w3c.dom.Node;
  21. import org.w3c.dom.NodeList;
  22.  
  23. public abstract class Osmdb {
  24. public String TABLE_NAME = "EXIST_osm";
  25. Connection hsqldb = null; // hsqldb DatabaseTool.openDb("database");
  26.  
  27. public Osmdb(Connection hsqldb, String tableName) {
  28. this.hsqldb = hsqldb;
  29. if (tableName != null) {
  30. this.TABLE_NAME = tableName;
  31. }
  32. }
  33. /**
  34. * 'HSQLDB.table.OSM_EXIST'を新規に作る
  35. * 'HSQLDB.table.AREA_NODE'を新規に作る
  36. * 既にテーブルが存在する時にはERROR
  37. * @throws SQLException
  38. */
  39. public void create() throws SQLException {
  40. String createSt;
  41. sql("DROP TABLE IF EXISTS "+ TABLE_NAME +" CASCADE");
  42. sql("DROP INDEX IF EXISTS "+ TABLE_NAME +"_index;");
  43. sql("DROP TABLE IF EXISTS AREA_NODE CASCADE");
  44.  
  45. // 'table.OSM_EXIST'を新規に作る
  46. createSt = "CREATE TABLE "+ TABLE_NAME +" (idref VARCHAR(12) NOT NULL, name VARCHAR(128), lat DOUBLE, lon DOUBLE, score INT, gmlid VARCHAR(12), area INT, PRIMARY KEY(idref), removed BOOLEAN DEFAULT FALSE NOT NULL);";
  47. Db.updateSQL(hsqldb, createSt);
  48. createSt = "CREATE INDEX "+ TABLE_NAME +"_index ON "+ TABLE_NAME +" (lat, lon);";
  49. Db.updateSQL(hsqldb, createSt);
  50.  
  51. // 'table.AREA_NODE'を新規に作る
  52. createSt = "CREATE TABLE AREA_NODE (idref VARCHAR(12) NOT NULL, pid VARCHAR(12), lat DOUBLE, lon DOUBLE);";
  53. Db.updateSQL(hsqldb, createSt);
  54. }
  55. void sql(String sql) throws SQLException {
  56. System.out.println(sql);
  57. try (PreparedStatement ps = hsqldb.prepareStatement(sql)) {
  58. ps.executeUpdate();
  59. }
  60. catch (SQLSyntaxErrorException e) {
  61. System.out.println("107:"+ e.toString());
  62. if (!(e.toString().startsWith("java.sql.SQLSyntaxErrorException: user lacks privilege or object not found:"))) {
  63. throw e;
  64. }
  65. }
  66. }
  67.  
  68. public static void outputWriter(PrintWriter pw, String text) {
  69. System.out.println("\t" + text);
  70. pw.print(text);
  71. }
  72. public static final int POINT_NO = 0; // 評価ポイント無し→ score=50
  73. public static final int POINT_FIXME = 1; // 評価ポイント|!(fixme=null) → score=1
  74. public static final int POINT_NO_BRAND = 2; // 評価ポイント|brand=null → score=1
  75. public static final int POINT_NO_NAME = 4; // 評価ポイント|name=null → score=1
  76. public static final int POINT_BUS_NO = 8; // 評価ポイント|!(bus=yes) → score=0
  77. /**
  78. * 'table.OSM_EXIST'の内容をCSV形式にして標準出力に出力する
  79. * @throws java.sql.SQLException
  80. */
  81. public void export() throws SQLException {
  82. String header = "idref,lat,lon,score";
  83. System.out.println("TABLE: "+ TABLE_NAME);
  84. System.out.println(header);
  85. PreparedStatement ps8 = hsqldb.prepareStatement("SELECT idref,lat,lon,score,removed FROM "+ TABLE_NAME);
  86. try (ResultSet rset8 = ps8.executeQuery()) {
  87. while (rset8.next()) {
  88. String idcode = rset8.getString(1);
  89. Double lat = rset8.getDouble(2);
  90. Double lon = rset8.getDouble(3);
  91. int score = rset8.getInt(4);
  92. boolean removed = rset8.getBoolean(5);
  93. String out = String.format("OSM: %s,%2.5f,%3.5f,%d,%b", idcode, lat, lon, score, removed);
  94. System.out.println(out);
  95. }
  96. }
  97. }
  98.  
  99. public int importExistingNode(Connection osmdb, ResultSet rset, int point) throws IOException, SQLException {
  100. String osmidStr = rset.getString("osm_id");
  101. String latStr = rset.getString("lat");
  102. String lonStr = rset.getString("lon");
  103. String brandStr = rset.getString("brand");
  104. String nameStr = rset.getString("name");
  105. String fixmeStr = "";
  106.  
  107. int score = 50;
  108. if (((point & POINT_NO_BRAND) != 0) && isNull(brandStr)) {
  109. score = 1;
  110. }
  111. if (((point & POINT_NO_NAME) != 0) && isNull(nameStr)) {
  112. score = 1;
  113. }
  114. boolean fixme = false;
  115. boolean busYes = false;
  116. if (((point & (POINT_FIXME | POINT_BUS_NO)) != 0)) {
  117. PreparedStatement ps8 = osmdb.prepareStatement("SELECT * FROM planet_osm_nodes where id='"+ osmidStr +"'");
  118. try (ResultSet rset8 = ps8.executeQuery()) {
  119. while (rset8.next()) {
  120. String tags = rset8.getString("tags");
  121. if (tags != null) {
  122. tags = tags.substring(1);
  123. tags = tags.substring(0, tags.length()-1);
  124. CsvRecord csv = new CsvRecord();
  125. csv.analizeRecord(tags);
  126. boolean key = true;
  127. boolean bus = false;
  128. for (String str : csv) {
  129. if (key) {
  130. if (str.startsWith("fixme")) {
  131. fixme = true;
  132. }
  133. if (str.equals("bus")) {
  134. bus = true;
  135. }
  136. key = false;
  137. }
  138. else {
  139. if (bus) {
  140. if (str.equals("yes")) {
  141. busYes = true;
  142. }
  143. bus = false;
  144. }
  145. key = true;
  146. }
  147. }
  148. System.out.println(tags);
  149. }
  150. }
  151. }
  152. if (((point & POINT_FIXME) != 0) && fixme) {
  153. score = 1;
  154. }
  155. if (((point & POINT_BUS_NO) != 0) && !busYes) {
  156. score = 0;
  157. }
  158. }
  159. if (score > 0) {
  160. // idref と brandStr をデータベースに格納する
  161. System.out.println("INSERT INTO "+ TABLE_NAME +" (idref,lat,lon,score,name) VALUES ("+ osmidStr +","+ latStr +","+ lonStr+","+ Integer.toString(score) +",'"+ nameStr +"')N");
  162. try (PreparedStatement ps5 = hsqldb.prepareStatement("INSERT INTO "+ TABLE_NAME +" (idref,lat,lon,score,name) VALUES (?,?,?,?,?)")) {
  163. ps5.setString(1, osmidStr);
  164. ps5.setDouble(2, Double.parseDouble(latStr));
  165. ps5.setDouble(3, Double.parseDouble(lonStr));
  166. ps5.setInt(4, score);
  167. ps5.setString(5, nameStr);
  168. ps5.executeUpdate();
  169. return 1;
  170. }
  171. catch (HsqlException | SQLIntegrityConstraintViolationException e) {
  172. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  173. // [HsqlException]は、無視する
  174. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  175. // [SQLIntegrityConstraintViolationException]は、無視する
  176. }
  177. }
  178.  
  179. return 0;
  180. }
  181.  
  182. /**
  183. * File(HttpPOST.EXIST_FILE)を読み取って、データベースに反映させる。<br>
  184. * その際に、OSMノードを評価し、scoreを算定する
  185. *
  186. * @param sqlWhere POI条件
  187. * @param point 評価ポイント[POINT_NO|POINT_BRAND|POINT_NAME] 1: 'brand' 2:'name'
  188. * @throws Exception エラー
  189. */
  190. public void readExisting(String sqlWhere, int point) throws Exception {
  191. Connection osmdb = null;
  192. long counter = 0L;
  193. try {
  194. osmdb = DatabaseTool.openDb("osmdb");
  195. String sqlSelect = "osm_id,brand,disused,name,ST_Y(ST_Transform(way,4326)) as lat,ST_X(ST_Transform(way,4326)) as lon";
  196. String sqlNode = String.format("SELECT %s FROM planet_osm_point %s", sqlSelect, sqlWhere);
  197. System.out.println(sqlNode);
  198. PreparedStatement ps1 = osmdb.prepareStatement(sqlNode);
  199. try (ResultSet rset1 = ps1.executeQuery()) {
  200. while (rset1.next()) {
  201. counter += importExistingNode(osmdb, rset1, point);
  202. }
  203. }
  204. sqlSelect = "osm_id,brand,disused,name,ST_Y(ST_Transform(ST_Centroid(way),4326)) as lat,ST_X(ST_Transform(ST_Centroid(way),4326)) as lon";
  205. String sqlArea = String.format("SELECT %s FROM planet_osm_polygon %s", sqlSelect, sqlWhere);
  206. System.out.println(sqlArea);
  207.  
  208. PreparedStatement ps2 = osmdb.prepareStatement(sqlArea);
  209. try (ResultSet rset2 = ps2.executeQuery()) {
  210. while (rset2.next()) {
  211. counter += importExistingNode(osmdb, rset2, point);
  212. }
  213. }
  214.  
  215. System.out.println("Exists Node count = " + counter);
  216. }
  217. finally {
  218. DatabaseTool.closeDb(osmdb);
  219. }
  220. }
  221.  
  222. /**
  223. *
  224. * SELECT
  225. * planet_osm_nodes.id,
  226. * planet_osm_nodes.tags
  227. * planet_osm_nodes.way --> lat & lon
  228. * FROM
  229. * public.planet_osm_point,
  230. * public.planet_osm_nodes
  231. * WHERE (
  232. * planet_osm_point.osm_id = planet_osm_nodes.id
  233. * and 'disused:amenity'=ANY(planet_osm_nodes.tags)
  234. * and 'fuel'=ANY(planet_osm_nodes.tags)
  235. * )
  236. *
  237. * @param kStr
  238. * @param vStr
  239. * @param point
  240. * @throws Exception
  241. */
  242. public void readExistingSub(String kStr, String vStr, int point) throws Exception {
  243. JsonBuilderFactory factory = Json.createBuilderFactory(null);
  244. JsonObjectBuilder builder = factory.createObjectBuilder();
  245. builder.add("k",kStr);
  246. builder.add("v", vStr);
  247. JsonObject tag = builder.build();
  248. readExistingSub(tag, point);
  249. }
  250. /**
  251. *
  252. * @param tag {k: string, v: string}
  253. * @param point
  254. * @throws Exception
  255. */
  256. public void readExistingSub(JsonObject tag,int point) throws Exception {
  257. boolean removed = true;
  258. Connection osmdb = null;
  259. long counter = 0L;
  260. try {
  261. osmdb = DatabaseTool.openDb("osmdb");
  262. String sqlSelect = "SELECT tags,id,ST_Y(ST_Transform(way,4326)) as lat,ST_X(ST_Transform(way,4326)) as lon";
  263. String sqlWhere = String.format("WHERE (osm_id=id AND '%s'=ANY(tags) AND '%s'=ANY(tags))", tag.getString("k"), tag.getString("v"));
  264. String sql = String.format("%s FROM planet_osm_point,planet_osm_nodes %s", sqlSelect, sqlWhere);
  265. System.out.println(sql);
  266. PreparedStatement ps1 = osmdb.prepareStatement(sql);
  267. try (ResultSet rset1 = ps1.executeQuery()) {
  268. while (rset1.next()) {
  269. String tags = rset1.getString("tags");
  270. String idref = rset1.getString("id");
  271. String latStr = rset1.getString("lat");
  272. String lonStr = rset1.getString("lon");
  273. double lat = Double.valueOf(latStr);
  274. double lon = Double.valueOf(lonStr);
  275. int score = score(point, tags);
  276. counter += insertExistingNode(idref, lat, lon, score, "", removed);
  277. }
  278. }
  279. System.out.println("Exists Node count = " + counter);
  280. }
  281. finally {
  282. DatabaseTool.closeDb(osmdb);
  283. }
  284. }
  285.  
  286. /**
  287. *
  288. * @param array tags: [{k: string, v: string}]
  289. * @param point
  290. * @throws Exception
  291. */
  292. public void readExistingSub(JsonArray array,int point) throws Exception {
  293. boolean removed = true;
  294. Connection osmdb = null;
  295. long counter = 0L;
  296. try {
  297. osmdb = DatabaseTool.openDb("osmdb");
  298. String sqlSelect = "SELECT tags,id,ST_Y(ST_Transform(way,4326)) as lat,ST_X(ST_Transform(way,4326)) as lon";
  299. String sqlSS = "";
  300. for (JsonValue v : array) {
  301. JsonObject tag = (JsonObject)v;
  302. sqlSS += String.format(" AND ('%s'=ANY(tags) AND '%s'=ANY(tags))", tag.getString("k"), tag.getString("v"));
  303. }
  304. String sqlWhere = String.format("WHERE (osm_id=id %s)", sqlSS);
  305. String sql = String.format("%s FROM planet_osm_point,planet_osm_nodes %s", sqlSelect, sqlWhere);
  306. System.out.println(sql);
  307. PreparedStatement ps1 = osmdb.prepareStatement(sql);
  308. try (ResultSet rset1 = ps1.executeQuery()) {
  309. while (rset1.next()) {
  310. String tags = rset1.getString("tags");
  311. String idref = rset1.getString("id");
  312. String latStr = rset1.getString("lat");
  313. String lonStr = rset1.getString("lon");
  314. double lat = Double.valueOf(latStr);
  315. double lon = Double.valueOf(lonStr);
  316. int score = score(point, tags);
  317. counter += insertExistingNode(idref, lat, lon, score, "", removed);
  318. }
  319. }
  320. System.out.println("Exists Node count = " + counter);
  321. }
  322. finally {
  323. DatabaseTool.closeDb(osmdb);
  324. }
  325. }
  326.  
  327. int score(int point, String tags) {
  328. int score = 50;
  329. if (tags == null) {
  330. return score;
  331. }
  332. boolean brandYes = false;
  333. boolean busYes = false;
  334. boolean fixmeYes = false;
  335. boolean nameYes = false;
  336. tags = tags.substring(1);
  337. tags = tags.substring(0, tags.length()-1);
  338. CsvRecord csv = new CsvRecord();
  339. csv.analizeRecord(tags);
  340.  
  341. boolean key = true;
  342. boolean bus = false;
  343. for (String str : csv) {
  344. if (key) {
  345. if (str.startsWith("fixme")) {
  346. fixmeYes = true;
  347. }
  348. else if (str.equals("bus")) {
  349. bus = true;
  350. }
  351. else if (str.equals("brand")) {
  352. brandYes = true;
  353. }
  354. else if (str.startsWith("name")) {
  355. nameYes = true;
  356. }
  357. key = false;
  358. }
  359. else {
  360. if (bus) {
  361. if (str.equals("yes")) {
  362. busYes = true;
  363. }
  364. bus = false;
  365. }
  366. key = true;
  367. }
  368. }
  369. if (((point & POINT_NO_BRAND) != 0) && !brandYes) {
  370. score = 1;
  371. }
  372. if (((point & POINT_NO_NAME) != 0) && !nameYes) {
  373. score = 1;
  374. }
  375. if (((point & POINT_FIXME) != 0) && fixmeYes) {
  376. score = 1;
  377. }
  378. if (((point & POINT_BUS_NO) != 0) && !busYes) {
  379. score = 0;
  380. }
  381. return score;
  382. }
  383. /**
  384. *
  385. * @param idref
  386. * @param lat
  387. * @param lon
  388. * @param score
  389. * @param name
  390. * @return
  391. * @throws IOException
  392. * @throws SQLException
  393. */
  394. public int insertExistingNode(String idref, double lat, double lon, int score, String name) throws IOException, SQLException {
  395. return insertExistingNode(idref, lat, lon, score, name, false);
  396. }
  397.  
  398. /**
  399. *
  400. * @param idref
  401. * @param lat
  402. * @param lon
  403. * @param score
  404. * @param name
  405. * @param removed
  406. * @return
  407. * @throws IOException
  408. * @throws SQLException
  409. */
  410. public int insertExistingNode(String idref, double lat, double lon, int score, String name, boolean removed) throws IOException, SQLException {
  411. // idref と brandStr をデータベースに格納する
  412. String sql = String.format("INSERT INTO %s (idref,lat,lon,score,name) VALUES (%s,%2.7f,%3.7f,%d,%s)",
  413. TABLE_NAME,
  414. idref, lat, lon, score, name);
  415. System.out.println(sql);
  416. sql = String.format("DELETE FROM %s WHERE idref=?", TABLE_NAME);
  417. try (PreparedStatement ps5 = hsqldb.prepareStatement(sql)) {
  418. ps5.setString(1, idref);
  419. ps5.executeUpdate();
  420. }
  421. catch (HsqlException | SQLIntegrityConstraintViolationException e) {
  422. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  423. // [HsqlException]は、無視する
  424. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  425. // [SQLIntegrityConstraintViolationException]は、無視する
  426. }
  427.  
  428. sql = String.format("INSERT INTO %s (idref,lat,lon,score,name,removed) VALUES (?,?,?,?,?,?)", TABLE_NAME);
  429. try (PreparedStatement ps5 = hsqldb.prepareStatement(sql)) {
  430. ps5.setString(1, idref);
  431. ps5.setDouble(2, lat);
  432. ps5.setDouble(3, lon);
  433. ps5.setInt(4, score);
  434. ps5.setString(5, name);
  435. ps5.setBoolean(6, removed);
  436. ps5.executeUpdate();
  437. return 1;
  438. }
  439. catch (HsqlException | SQLIntegrityConstraintViolationException e) {
  440. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  441. // [HsqlException]は、無視する
  442. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  443. // [SQLIntegrityConstraintViolationException]は、無視する
  444. }
  445. return 0;
  446. }
  447. boolean isNull(String vstr) {
  448. if (vstr == null) {
  449. return true;
  450. }
  451. return (vstr.trim().length() < 1);
  452. }
  453. int readAreaNodes(Connection con, Node node) throws IOException, SQLException {
  454. int iCounter = 0;
  455.  
  456. NodeList nodes = node.getChildNodes();
  457. for (int i = 0; i < nodes.getLength(); i++) {
  458. Node node2 = nodes.item(i);
  459. switch (node2.getNodeName()) {
  460. case "node":
  461. iCounter++;
  462. importAreaNode(con, node2);
  463. break;
  464. default:
  465. iCounter += readAreaNodes(con, node2);
  466. break;
  467. }
  468. }
  469. return iCounter;
  470. }
  471. void importAreaNode(Connection con, Node node) throws IOException, SQLException {
  472. String idrefStr = "";
  473. String latStr = "";
  474. String lonStr = "";
  475.  
  476. NodeList nodes = node.getChildNodes();
  477. if (nodes.getLength() > 0) {
  478. return;
  479. }
  480.  
  481. NamedNodeMap nodeMap = node.getAttributes();
  482. if (nodeMap != null) {
  483. for (int j=0; j < nodeMap.getLength(); j++) {
  484. switch (nodeMap.item(j).getNodeName()) {
  485. case "id":
  486. idrefStr = nodeMap.item(j).getNodeValue();
  487. break;
  488. case "lat":
  489. latStr = nodeMap.item(j).getNodeValue();
  490. break;
  491. case "lon":
  492. lonStr = nodeMap.item(j).getNodeValue();
  493. break;
  494. default:
  495. break;
  496. }
  497. }
  498. // idref と brandStr をデータベースに格納する
  499. System.out.println("insert into AREA_NODE(idref,lat,lon) values("+ idrefStr +","+ latStr +","+ lonStr+")");
  500. try (PreparedStatement ps5 = con.prepareStatement("INSERT INTO AREA_NODE (idref,lat,lon) VALUES (?,?,?)")) {
  501. ps5.setString(1, idrefStr);
  502. ps5.setDouble(2, Double.parseDouble(latStr));
  503. ps5.setDouble(3, Double.parseDouble(lonStr));
  504. ps5.executeUpdate();
  505. }
  506. }
  507. }
  508. static Position getNdPosition(Connection con, String idref) throws SQLException {
  509. PreparedStatement ps8 = con.prepareStatement("SELECT lat,lon FROM AREA_NODE where idref=?");
  510. ps8.setString(1, idref);
  511. try (ResultSet rset8 = ps8.executeQuery()) {
  512. while (rset8.next()) {
  513. Double lat = rset8.getDouble(1);
  514. Double lon = rset8.getDouble(2);
  515. return new Position(lat,lon);
  516. }
  517. }
  518. return null;
  519. }
  520. }