Newer
Older
osmCoverage / src / osm / jp / api / Osmdb.java
@hayashi hayashi on 23 Nov 2017 21 KB DbExistの開発着手
  1. package osm.jp.api;
  2.  
  3. import java.net.*;
  4. import java.util.List;
  5. import java.util.Map;
  6. import java.util.concurrent.TimeUnit;
  7. import java.io.*;
  8. import java.sql.Connection;
  9. import java.sql.PreparedStatement;
  10. import java.sql.ResultSet;
  11. import java.sql.SQLException;
  12. import java.sql.SQLIntegrityConstraintViolationException;
  13. import java.sql.SQLSyntaxErrorException;
  14. import javax.xml.parsers.DocumentBuilder;
  15. import javax.xml.parsers.DocumentBuilderFactory;
  16. import javax.xml.parsers.ParserConfigurationException;
  17. import org.hsqldb.HsqlException;
  18. import org.w3c.dom.NamedNodeMap;
  19. import org.w3c.dom.Node;
  20. import org.w3c.dom.NodeList;
  21. import org.xml.sax.SAXException;
  22.  
  23. public abstract class Osmdb {
  24. public static String host = "http://overpass-api.de";
  25. public static final String EXIST_FILE = "exist.osm.xml";
  26. public static final String TABLE_NAME = "OSM_EXIST";
  27.  
  28. /*
  29. public static void main(String[] args) throws MalformedURLException, ProtocolException, IOException {
  30. double minlat = 35.13d;
  31. double maxlat = 35.66d;
  32. double minlon = 138.99d;
  33. double maxlon = 139.79d;
  34. //getCapabilities(new File("output.xml"), "highway", "bus_stop", minlat, maxlat, minlon, maxlon);
  35. //getCapabilities(new File("output.xml"), "highway", "disused:bus_stop", minlat, maxlat, minlon, maxlon);
  36. //getCapabilities(new File("output.xml"), "amenity", "bus_station", minlat, maxlat, minlon, maxlon);
  37. //getCapabilities(new File("output.xml"), "public_transport", "platform", minlat, maxlat, minlon, maxlon);
  38. getCapabilities("public_transport", "stop_position", minlat, maxlat, minlon, maxlon, "node");
  39. getCapabilities("amenity", "fuel", minlat, maxlat, minlon, maxlon, "way");
  40. }
  41. */
  42. /**
  43. * 'HSQLDB.table.OSM_EXIST'を新規に作る
  44. * 'HSQLDB.table.AREA_NODE'を新規に作る
  45. * 既にテーブルが存在する時にはERROR
  46. * @param con
  47. * @throws SQLException
  48. */
  49. public static void create(Connection con) throws SQLException {
  50. String createSt;
  51. sql(con, "DROP TABLE IF EXISTS "+ Osmdb.TABLE_NAME +" CASCADE");
  52. //sql(con, "DROP INDEX "+ HttpPOST.TABLE_NAME +"_index;");
  53. sql(con, "DROP TABLE IF EXISTS AREA_NODE CASCADE");
  54.  
  55. // 'table.FUEL_EXIST'を新規に作る
  56. createSt = "CREATE TABLE "+ Osmdb.TABLE_NAME +" (idref VARCHAR(12) NOT NULL, name VARCHAR(128), lat DOUBLE, lon DOUBLE, score INT, PRIMARY KEY(idref));";
  57. Db.updateSQL(con, createSt);
  58. createSt = "CREATE INDEX "+ Osmdb.TABLE_NAME +"_index ON "+ Osmdb.TABLE_NAME +" (lat, lon);";
  59. Db.updateSQL(con, createSt);
  60.  
  61. // 'table.AREA_NODE'を新規に作る
  62. createSt = "CREATE TABLE AREA_NODE (idref VARCHAR(12) NOT NULL, pid VARCHAR(12), lat DOUBLE, lon DOUBLE);";
  63. Db.updateSQL(con, createSt);
  64. }
  65. public static void sql(Connection con, String sql) throws SQLException {
  66. System.out.println(sql);
  67. try (PreparedStatement ps = con.prepareStatement(sql)) {
  68. ps.executeUpdate();
  69. }
  70. catch (SQLSyntaxErrorException e) {
  71. System.out.println("107:"+ e.toString());
  72. if (!(e.toString().startsWith("java.sql.SQLSyntaxErrorException: user lacks privilege or object not found:"))) {
  73. throw e;
  74. }
  75. }
  76. }
  77.  
  78. /*
  79. */
  80. public static void getCapabilities(String key, String value, double minLat, double maxLat, double minLon, double maxLon) throws MalformedURLException, ProtocolException, IOException {
  81. getCapabilities(key, value, minLat, maxLat, minLon, maxLon, "node");
  82. }
  83.  
  84. public static void getCapabilities(String key, String value, double minLat, double maxLat, double minLon, double maxLon, String type) throws MalformedURLException, ProtocolException, IOException {
  85. StringBuilder queryText = new StringBuilder();
  86. queryText.append("<osm-script timeout=\"900\" element-limit=\"1073741824\">");
  87. queryText.append(" <union>");
  88. queryText.append(" <query type=\""+ type +"\">");
  89. queryText.append(" <has-kv k=\""+ key +"\" v=\""+ value +"\"/>");
  90. queryText.append(" <bbox-query s=\"" + minLat + "\" n=\"" + maxLat + "\" w=\"" + minLon + "\" e=\"" + maxLon + "\"/>");
  91. queryText.append(" </query>");
  92. queryText.append(" </union>");
  93. queryText.append(" <print/>");
  94. queryText.append("</osm-script>");
  95. getQuery(queryText.toString());
  96. }
  97.  
  98. /**
  99. *
  100. * @param queryText クエリテキスト(Overpass_API/Overpass_QL)
  101. * @throws MalformedURLException
  102. * @throws ProtocolException
  103. * @throws IOException
  104. */
  105. public static void getQuery(String queryText) throws MalformedURLException, ProtocolException, IOException {
  106. System.out.println(host + "/api/interpreter");
  107. URL url = new URL(host + "/api/interpreter");
  108. int responsecode = 0;
  109.  
  110. do {
  111. HttpURLConnection urlconn = (HttpURLConnection)url.openConnection();
  112. try {
  113. urlconn.setRequestMethod("POST");
  114. urlconn.setDoOutput(true); // POSTのデータを後ろに付ける
  115. urlconn.setInstanceFollowRedirects(false); // 勝手にリダイレクトさせない
  116. urlconn.setRequestProperty("Accept-Language", "ja;q=0.7,en;q=0.3");
  117. urlconn.setRequestProperty("Content-Type","text/xml;charset=utf-8");
  118. urlconn.connect();
  119.  
  120. try (PrintWriter pw = new PrintWriter(new BufferedWriter(new OutputStreamWriter(urlconn.getOutputStream(), "utf-8")))) {
  121. outputWriter(pw, queryText);
  122. pw.flush();
  123. }
  124.  
  125. try {
  126. TimeUnit.SECONDS.sleep(1);
  127. } catch (InterruptedException e) {}
  128.  
  129. responsecode = urlconn.getResponseCode();
  130. System.out.println("レスポンスコード[" + responsecode + "] " +
  131. "レスポンスメッセージ[" + urlconn.getResponseMessage() + "]");
  132. Map<String,List<String>> headers = urlconn.getHeaderFields();
  133. for (Map.Entry<String, List<String>> bar : headers.entrySet()) {
  134. System.out.print("\t" + bar.getKey() +"\t: "); // キーを取得
  135. List<String> vals = bar.getValue(); // 値を取得
  136. for(String str : vals) {
  137. System.out.print("["+ str +"],");
  138. }
  139. System.out.println();
  140. }
  141. if ((responsecode == 429) || (responsecode == 504) || (responsecode == 500)) {
  142. // レスポンスコード[429] レスポンスメッセージ[Too Many Requests]
  143. // レスポンスコード[500] レスポンスメッセージ[Internal server error]
  144. // レスポンスコード[504] レスポンスメッセージ[Gateway Timeout]
  145. System.out.print("Waite 5 minites.");
  146. try {
  147. TimeUnit.MINUTES.sleep(5);
  148. } catch (InterruptedException e) {}
  149. }
  150. else {
  151. System.out.println("\n---- ボディ ----");
  152.  
  153. File oFile = new File(Osmdb.EXIST_FILE);
  154. oFile.deleteOnExit();
  155. try (
  156. BufferedWriter hw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(oFile), "UTF-8"));
  157. BufferedReader reader = new BufferedReader(new InputStreamReader(urlconn.getInputStream(), "UTF-8"))
  158. ) {
  159. while (true) {
  160. String line = reader.readLine();
  161. if (line == null) {
  162. break;
  163. }
  164. hw.write(line);
  165. hw.newLine();
  166. }
  167. hw.flush();
  168. }
  169. }
  170. }
  171. catch (java.net.ConnectException e) {
  172. // レスポンスコード[600] レスポンスメッセージ[接続がタイムアウトしました (Connection timed out)]
  173. responsecode = 600;
  174. }
  175. finally {
  176. urlconn.disconnect();
  177. }
  178. System.out.print("Waite 5 seconds.");
  179. try {
  180. TimeUnit.SECONDS.sleep(5);
  181. } catch (InterruptedException e) {}
  182. }
  183. while ((responsecode == 429) || (responsecode == 504) || (responsecode == 600));
  184. }
  185. public static void outputWriter(PrintWriter pw, String text) {
  186. System.out.println("\t" + text);
  187. pw.print(text);
  188. }
  189. public static final int POINT_NO = 0; // 評価ポイント無し→ score=50
  190. public static final int POINT_FIXME = 1; // 評価ポイント無し→ score=50
  191. public static final int POINT_BRAND = 2; // 評価ポイント|brand=null → score=1
  192. public static final int POINT_NAME = 4; // 評価ポイント|name=null → score=1
  193. /**
  194. * File(HttpPOST.EXIST_FILE)を読み取って、データベースに反映させる。<br>
  195. * その際に、OSMノードを評価し、scoreを算定する
  196. * @param con 反映先のデータベースコネクタ(HSQLDB)
  197. * @param point 評価ポイント[POINT_NO|POINT_BRAND|POINT_NAME] 1: 'brand' 2:'name'
  198. * @throws FileNotFoundException File(HttpPOST.EXIST_FILE)が存在しない
  199. * @throws ClassNotFoundException
  200. * @throws SQLException データベースエラー
  201. * @throws IOException
  202. * @throws ParserConfigurationException
  203. * @throws SAXException
  204. */
  205. public void readExistingFile (Connection con, int point) throws FileNotFoundException, ClassNotFoundException, SQLException, IOException, ParserConfigurationException, SAXException {
  206.  
  207. DocumentBuilderFactory factory;
  208. DocumentBuilder builder;
  209.  
  210. factory = DocumentBuilderFactory.newInstance();
  211. builder = factory.newDocumentBuilder();
  212. factory.setIgnoringElementContentWhitespace(true);
  213. factory.setIgnoringComments(true);
  214. factory.setValidating(true);
  215.  
  216. Node root = builder.parse(new File(Osmdb.EXIST_FILE));
  217.  
  218. readAreaNodes(con, root);
  219. int iCounter = readExistingNodes(con, root, point);
  220. System.out.println("既存ノード数["+ iCounter +"]");
  221. }
  222. public int readAreaNodes(Connection con, Node node) throws IOException, SQLException {
  223. int iCounter = 0;
  224.  
  225. NodeList nodes = node.getChildNodes();
  226. for (int i = 0; i < nodes.getLength(); i++) {
  227. Node node2 = nodes.item(i);
  228. switch (node2.getNodeName()) {
  229. case "node":
  230. iCounter++;
  231. importAreaNode(con, node2);
  232. break;
  233. default:
  234. iCounter += readAreaNodes(con, node2);
  235. break;
  236. }
  237. }
  238. return iCounter;
  239. }
  240. public void importAreaNode(Connection con, Node node) throws IOException, SQLException {
  241. String idrefStr = "";
  242. String latStr = "";
  243. String lonStr = "";
  244.  
  245. NodeList nodes = node.getChildNodes();
  246. if (nodes.getLength() > 0) {
  247. return;
  248. }
  249.  
  250. NamedNodeMap nodeMap = node.getAttributes();
  251. if (nodeMap != null) {
  252. for (int j=0; j < nodeMap.getLength(); j++) {
  253. switch (nodeMap.item(j).getNodeName()) {
  254. case "id":
  255. idrefStr = nodeMap.item(j).getNodeValue();
  256. break;
  257. case "lat":
  258. latStr = nodeMap.item(j).getNodeValue();
  259. break;
  260. case "lon":
  261. lonStr = nodeMap.item(j).getNodeValue();
  262. break;
  263. default:
  264. break;
  265. }
  266. }
  267. // idref と brandStr をデータベースに格納する
  268. System.out.println("insert into AREA_NODE(idref,lat,lon) values("+ idrefStr +","+ latStr +","+ lonStr+")");
  269. try (PreparedStatement ps5 = con.prepareStatement("INSERT INTO AREA_NODE (idref,lat,lon) VALUES (?,?,?)")) {
  270. ps5.setString(1, idrefStr);
  271. ps5.setDouble(2, Double.parseDouble(latStr));
  272. ps5.setDouble(3, Double.parseDouble(lonStr));
  273. ps5.executeUpdate();
  274. }
  275. }
  276. }
  277.  
  278. public int readExistingNodes(Connection con, Node node, int point) throws IOException, SQLException {
  279. int iCounter = 0;
  280.  
  281. NodeList nodes = node.getChildNodes();
  282. for (int i = 0; i < nodes.getLength(); i++) {
  283. Node node2 = nodes.item(i);
  284. switch (node2.getNodeName()) {
  285. case "node":
  286. iCounter++;
  287. importExistingNode(con, node2, point);
  288. break;
  289. case "way":
  290. iCounter++;
  291. importExistingArea(con, node2, point);
  292. break;
  293. default:
  294. iCounter += readExistingNodes(con, node2, point);
  295. break;
  296. }
  297. }
  298. return iCounter;
  299. }
  300.  
  301. public void importExistingNode(Connection con, Node node, int point) throws IOException, SQLException {
  302. String idrefStr = "";
  303. String latStr = "";
  304. String lonStr = "";
  305. String brandStr = "";
  306. String nameStr = "";
  307. String fixmeStr = "";
  308.  
  309. NamedNodeMap nodeMap = node.getAttributes();
  310. if (nodeMap != null) {
  311. for (int j=0; j < nodeMap.getLength(); j++) {
  312. switch (nodeMap.item(j).getNodeName()) {
  313. case "id":
  314. idrefStr = nodeMap.item(j).getNodeValue();
  315. break;
  316. case "lat":
  317. latStr = nodeMap.item(j).getNodeValue();
  318. break;
  319. case "lon":
  320. lonStr = nodeMap.item(j).getNodeValue();
  321. break;
  322. default:
  323. break;
  324. }
  325. }
  326.  
  327. NodeList nodes = node.getChildNodes();
  328. if (nodes.getLength() == 0) {
  329. return;
  330. }
  331. for (int i = 0; i < nodes.getLength(); i++) {
  332. Node node2 = nodes.item(i);
  333. if (node2.getNodeName().equals("tag")) {
  334. OsmnodeTag tagnode = new OsmnodeTag(node2);
  335. String value = tagnode.getValue("brand");
  336. if (value != null) {
  337. brandStr = value;
  338. }
  339. value = tagnode.getValue("name");
  340. if (value != null) {
  341. nameStr = value;
  342. }
  343. value = tagnode.getValue("name:ja");
  344. if (value != null) {
  345. nameStr = value;
  346. }
  347. value = tagnode.getValue("fixme");
  348. if (value != null) {
  349. fixmeStr = value;
  350. }
  351. }
  352. }
  353.  
  354. int score = 50;
  355. if (((point & POINT_BRAND) != 0) && brandStr.equals("")) {
  356. score = 1;
  357. }
  358. if (((point & POINT_NAME) != 0) && brandStr.equals("")) {
  359. score = 1;
  360. }
  361. if (((point & POINT_FIXME) != 0) && !fixmeStr.equals("")) {
  362. score = 1;
  363. }
  364. // idref と brandStr をデータベースに格納する
  365. System.out.println("INSERT INTO "+ TABLE_NAME +" (idref,lat,lon,score,name) VALUES ("+ idrefStr +","+ latStr +","+ lonStr+","+ Integer.toString(score) +",'"+ nameStr +"')N");
  366. try (PreparedStatement ps5 = con.prepareStatement("INSERT INTO "+ TABLE_NAME +" (idref,lat,lon,score,name) VALUES (?,?,?,?,?)")) {
  367. ps5.setString(1, idrefStr);
  368. ps5.setDouble(2, Double.parseDouble(latStr));
  369. ps5.setDouble(3, Double.parseDouble(lonStr));
  370. ps5.setInt(4, score);
  371. ps5.setString(5, nameStr);
  372. ps5.executeUpdate();
  373. }
  374. catch (HsqlException | SQLIntegrityConstraintViolationException e) {
  375. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  376. // [HsqlException]は、無視する
  377. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  378. // [SQLIntegrityConstraintViolationException]は、無視する
  379. }
  380. }
  381. }
  382.  
  383. public void importExistingArea(Connection con, Node node, int point) throws IOException, SQLException {
  384. String idrefStr = "";
  385. double maxlat = -90.0D;
  386. double maxlon = -180.0D;
  387. double minlat = 90.0D;
  388. double minlon = 180.0D;
  389. String nameStr = "";
  390. String brandStr = "";
  391. String fixmeStr = "";
  392.  
  393. NamedNodeMap nodeMap = node.getAttributes();
  394. if (nodeMap != null) {
  395. for (int j=0; j < nodeMap.getLength(); j++) {
  396. switch (nodeMap.item(j).getNodeName()) {
  397. case "id":
  398. idrefStr = nodeMap.item(j).getNodeValue();
  399. break;
  400. default:
  401. break;
  402. }
  403. }
  404.  
  405. NodeList nodes = node.getChildNodes();
  406. for (int i = 0; i < nodes.getLength(); i++) {
  407. Node node2 = nodes.item(i);
  408. if (node2.getNodeName().equals("tag")) {
  409. OsmnodeTag tagnode = new OsmnodeTag(node2);
  410. String value = tagnode.getValue("brand");
  411. if (value != null) {
  412. brandStr = value;
  413. }
  414. value = tagnode.getValue("name:ja");
  415. if (value != null) {
  416. nameStr = value;
  417. }
  418. value = tagnode.getValue("name");
  419. if (value != null) {
  420. nameStr = value;
  421. }
  422. value = tagnode.getValue("fixme");
  423. if (value != null) {
  424. fixmeStr = value;
  425. }
  426. }
  427. else if (node2.getNodeName().equals("nd")) {
  428. OsmnodeNd ndnode = new OsmnodeNd(node2);
  429. String ref = ndnode.getRef();
  430. Position pos = getNdPosition(con, ref);
  431. if (pos != null) {
  432. minlat = (pos.lat < minlat ? pos.lat : minlat);
  433. minlon = (pos.lon < minlon ? pos.lon : minlon);
  434. maxlat = (pos.lat > maxlat ? pos.lat : maxlat);
  435. maxlon = (pos.lon > maxlon ? pos.lon : maxlon);
  436. }
  437. }
  438. }
  439. if ((maxlat != -90.0D) && (maxlon != -180.0D) && (minlon != 180.0D) && (minlat != 90.0D)) {
  440. double lat = (maxlat + minlat) / 2;
  441. double lon = (maxlon + minlon) / 2;
  442. int score = 50;
  443.  
  444. if (((point & POINT_BRAND) != 0) && brandStr.equals("")) {
  445. score = 1;
  446. }
  447. if (((point & POINT_NAME) != 0) && nameStr.equals("")) {
  448. score = 1;
  449. }
  450. if (((point & POINT_FIXME) != 0) && !fixmeStr.equals("")) {
  451. score = 1;
  452. }
  453.  
  454. // idref と nameStr をデータベースに格納する
  455. System.out.println("INSERT INTO "+ TABLE_NAME +" (idref,lat,lon,score,name) VALUES("+ idrefStr +","+ lat +","+ lon+","+ Integer.toString(score) +",'"+ nameStr +"');");
  456. try (PreparedStatement ps5 = con.prepareStatement("INSERT INTO "+ TABLE_NAME +" (idref,lat,lon,score,name) VALUES (?,?,?,?,?)")) {
  457. ps5.setString(1, idrefStr);
  458. ps5.setDouble(2, lat);
  459. ps5.setDouble(3, lon);
  460. ps5.setInt(4, score);
  461. ps5.setString(5, nameStr);
  462. ps5.executeUpdate();
  463. }
  464. catch (HsqlException | SQLIntegrityConstraintViolationException e) {
  465. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  466. // [HsqlException]は、無視する
  467. // integrity constraint violation: unique constraint or index violation; SYS_PK_10069 table: FUEL_EXIST
  468. // [SQLIntegrityConstraintViolationException]は、無視する
  469. }
  470. }
  471. }
  472. }
  473. public static Position getNdPosition(Connection con, String idref) throws SQLException {
  474. PreparedStatement ps8 = con.prepareStatement("SELECT lat,lon FROM AREA_NODE where idref=?");
  475. ps8.setString(1, idref);
  476. try (ResultSet rset8 = ps8.executeQuery()) {
  477. while (rset8.next()) {
  478. Double lat = rset8.getDouble(1);
  479. Double lon = rset8.getDouble(2);
  480. return new Position(lat,lon);
  481. }
  482. }
  483. return null;
  484. }
  485. }