diff --git a/LICENSE.txt b/LICENSE.txt index 942e6bb..484315f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -21,17 +21,3 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------- -`talkCrawler.jar` -Copyright (c) 2019 Yuu Hayashi -This software is released under the MIT License, see LICENSE.txt. - -------------------------------------------------------------------- - -* [hayashi.jar](https://osdn.net/projects/hayashilib/) -Copyright (c) 2013 Yuu Hayashi -This software is released under the MIT License. - -* [javax.json.jar](https://javaee.github.io/jsonp/) - -* [postgresql-42.2.4.jar](https://www.postgresql.org/) diff --git a/README.md b/README.md index 780508f..3e32095 100644 --- a/README.md +++ b/README.md @@ -10,4 +10,14 @@ ## License -* [MIT license](LICENSE.txt) +This software is released under the MIT License, see LICENSE.txt. + +------------------------------------------------------------------- +`talkCrawler.jar` +Copyright (c) 2019 Yuu Hayashi +This software is released under the MIT License, see LICENSE.txt. + +------------------------------------------------------------------- + +* [javax.json.jar](https://javaee.github.io/jsonp/) + diff --git a/src/talkCrawler.properties b/src/talkCrawler.properties new file mode 100644 index 0000000..c57f852 --- /dev/null +++ b/src/talkCrawler.properties @@ -0,0 +1,6 @@ +# To change this license header, choose License Headers in Project Properties. +# To change this template file, choose Tools | Templates +# and open the template in the editor. + +SOLR_URL=http://surveyor.mydns.jp/solr/talkja/update/json?commit=true +MONTHLY_INDEX=https://lists.openstreetmap.org/pipermail/talk-ja \ No newline at end of file diff --git a/src/talkcrawler/Article.java b/src/talkcrawler/Article.java index c6e10cc..c661301 100644 --- a/src/talkcrawler/Article.java +++ b/src/talkcrawler/Article.java @@ -17,7 +17,6 @@ import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; -import tool.http.Post; public class Article { String url; @@ -31,18 +30,6 @@ SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss' UTC'"); - @SuppressWarnings({"UseSpecificCatch", "CallToPrintStackTrace"}) - public static void main(String[] args) { - try { - Article ins = new Article(MonthlyIndex.MONTHLY_INDEX, "2019-January", "010424"); - ins.load(); - Post.post(Article.toJsonArray(ins.toJsonObject())); - } - catch (Exception e) { - e.printStackTrace(); - } - } - /** * * @param path @@ -173,7 +160,6 @@ date = sdf3.parse(date1 + date3); } catch (ParseException e2) { - e2.printStackTrace(); throw new IOException(e2); } } diff --git a/src/talkcrawler/DailyIndex.java b/src/talkcrawler/DailyIndex.java index 06d4117..fa49e6e 100644 --- a/src/talkcrawler/DailyIndex.java +++ b/src/talkcrawler/DailyIndex.java @@ -20,7 +20,14 @@ String path; String dir; String file; + TalkCrawlerProperties prop; + /** + * COMMAND + * + * @param args + * @throws IOException + */ public static void main(String[] args) throws IOException { Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); @@ -29,20 +36,22 @@ DateFormat df = new SimpleDateFormat("yyyy-MMMMMMMM", Locale.UK); df.setTimeZone(TimeZone.getTimeZone("GMT")); String monthly = df.format(calendar.getTime()); - - DailyIndex ins = new DailyIndex(MonthlyIndex.MONTHLY_INDEX, monthly, "date.html"); + + TalkCrawlerProperties prop = new TalkCrawlerProperties().load(); + DailyIndex ins = new DailyIndex(prop, monthly, "date.html"); ins.load(); } - + /** * - * @param path + * @param prop MonthlyIndex.MONTHLY_INDEX * @param dir * @param file * @throws java.io.IOException */ - public DailyIndex(String path, String dir, String file) throws IOException { - this.path = path; + public DailyIndex(TalkCrawlerProperties prop, String dir, String file) throws IOException { + this.prop = prop; + this.path = prop.getProperty("MONTHLY_INDEX"); this.dir = dir; this.file = file; } @@ -116,9 +125,9 @@ } if (str.equals("")) { System.out.println("-----"); - Article article = new Article(MonthlyIndex.MONTHLY_INDEX, dir, id); + Article article = new Article(path, dir, id); article.load(); - Post.post(Article.toJsonArray(article.toJsonObject())); + new Post(prop).post(Article.toJsonArray(article.toJsonObject())); } if (datain && str.toUpperCase().startsWith("")) { break; diff --git a/src/talkcrawler/MonthlyIndex.java b/src/talkcrawler/MonthlyIndex.java index 4e30308..53c9b99 100644 --- a/src/talkcrawler/MonthlyIndex.java +++ b/src/talkcrawler/MonthlyIndex.java @@ -10,25 +10,28 @@ import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArrayBuilder; -import tool.http.Post; public class MonthlyIndex { - final static String MONTHLY_INDEX = "https://lists.openstreetmap.org/pipermail/talk-ja"; + //final static String MONTHLY_INDEX = "https://lists.openstreetmap.org/pipermail/talk-ja"; + TalkCrawlerProperties prop; JsonArrayBuilder arryBuild; + URL monthlyIndexUrl; /** * + * @param prop * @throws java.io.IOException */ - public MonthlyIndex() throws IOException { + public MonthlyIndex(TalkCrawlerProperties prop) throws IOException { + this.prop = prop; this.arryBuild = Json.createArrayBuilder(); + this.monthlyIndexUrl = new URL(prop.getProperty("MONTHLY_INDEX")); } @SuppressWarnings({"CallToPrintStackTrace", "SleepWhileInLoop", "UseSpecificCatch"}) public void load() { try { - URL url = new URL(MONTHLY_INDEX); - HttpURLConnection http = (HttpURLConnection)url.openConnection(); + HttpURLConnection http = (HttpURLConnection)monthlyIndexUrl.openConnection(); http.setRequestMethod("GET"); http.connect(); @@ -41,6 +44,7 @@ } } + void getHtml(BufferedReader reader) throws IOException { boolean in = false; String line; @@ -125,11 +129,11 @@ name = st.nextToken().trim(); } - System.out.println(String.format("%s/%s/%s", MONTHLY_INDEX, dir, name)); - DailyIndex daily = new DailyIndex(MONTHLY_INDEX, dir, name); + System.out.println(String.format("./%s/%s", dir, name)); + DailyIndex daily = new DailyIndex(prop, dir, name); daily.load(); } } - Post.post(arryBuild.build()); + //new Post().post(arryBuild.build()); } } diff --git a/src/talkcrawler/TalkCrawler.java b/src/talkcrawler/TalkCrawler.java index 92b34fc..6ce4d30 100644 --- a/src/talkcrawler/TalkCrawler.java +++ b/src/talkcrawler/TalkCrawler.java @@ -5,7 +5,8 @@ @SuppressWarnings({"UseSpecificCatch", "CallToPrintStackTrace"}) public static void main(String[] args) { try { - MonthlyIndex ins = new MonthlyIndex(); + TalkCrawlerProperties prop = new TalkCrawlerProperties().load(); + MonthlyIndex ins = new MonthlyIndex(prop); ins.load(); } catch (Exception e) { diff --git a/src/talkcrawler/TalkCrawlerProperties.java b/src/talkcrawler/TalkCrawlerProperties.java new file mode 100644 index 0000000..6bdfa2d --- /dev/null +++ b/src/talkcrawler/TalkCrawlerProperties.java @@ -0,0 +1,21 @@ +package talkcrawler; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class TalkCrawlerProperties extends Properties { + public TalkCrawlerProperties() { + super(); + } + + public TalkCrawlerProperties load() throws FileNotFoundException, IOException { + try (InputStream inStream = new BufferedInputStream(new FileInputStream("talkCrawler.properties"))) { + this.load(inStream); + } + return this; + } +} diff --git a/src/tool/http/Post.java b/src/tool/http/Post.java index a32fd90..2c8d7b5 100644 --- a/src/tool/http/Post.java +++ b/src/tool/http/Post.java @@ -1,17 +1,24 @@ package tool.http; +import java.io.BufferedInputStream; import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintStream; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; +import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; import javax.json.JsonArray; import javax.json.JsonObject; import talkcrawler.Article; +import talkcrawler.TalkCrawlerProperties; import tool.json.JsonTool; /** @@ -19,11 +26,17 @@ * @author yuu */ public class Post { - public static void post(JsonArray array) { + URL url; + + public Post(TalkCrawlerProperties prop) throws MalformedURLException, FileNotFoundException, IOException { + this.url = new URL(prop.getProperty("SOLR_URL")); + } + + @SuppressWarnings("UseSpecificCatch") + public void post(JsonArray array) { String jsonText = array.toString(); HttpURLConnection con = null; try { - URL url = new URL("http://surveyor.mydns.jp/solr/talkja/update/json?commit=true"); con = (HttpURLConnection) url.openConnection(); con.setConnectTimeout(60000); con.setReadTimeout(60000); @@ -40,16 +53,15 @@ // recv response int statusCode = con.getResponseCode(); - String responseData = ""; StringBuilder sb = new StringBuilder(); try (InputStream stream = con.getInputStream()) { - String line = ""; + String line; BufferedReader br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); while ((line = br.readLine()) != null) { sb.append(line); } } - responseData = sb.toString(); + String responseData = sb.toString(); JsonObject res = JsonTool.parse(responseData); if (res != null) { JsonObject header = res.getJsonObject("responseHeader"); diff --git a/talkCrawler.properties b/talkCrawler.properties new file mode 100644 index 0000000..c57f852 --- /dev/null +++ b/talkCrawler.properties @@ -0,0 +1,6 @@ +# To change this license header, choose License Headers in Project Properties. +# To change this template file, choose Tools | Templates +# and open the template in the editor. + +SOLR_URL=http://surveyor.mydns.jp/solr/talkja/update/json?commit=true +MONTHLY_INDEX=https://lists.openstreetmap.org/pipermail/talk-ja \ No newline at end of file diff --git a/test/talkcrawler/ArticleTest.java b/test/talkcrawler/ArticleTest.java index 5f44a84..8f15fed 100644 --- a/test/talkcrawler/ArticleTest.java +++ b/test/talkcrawler/ArticleTest.java @@ -1,20 +1,30 @@ package talkcrawler; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; import static junit.framework.TestCase.fail; import org.junit.Test; import tool.http.Post; public class ArticleTest { - /** - * - */ + TalkCrawlerProperties prop = new TalkCrawlerProperties(); + + protected void setUp(){ + try { + prop.load(); + } catch (IOException ex) { + Logger.getLogger(DailyIndexTest.class.getName()).log(Level.SEVERE, null, ex); + } + } + @Test @SuppressWarnings("UseSpecificCatch") public void test2019_01_010424() { try { - Article ins = new Article(MonthlyIndex.MONTHLY_INDEX, "2019-January", "010424"); + Article ins = new Article(prop.getProperty("MONTHLY_INDEX"), "2019-January", "010424"); ins.load(); - Post.post(Article.toJsonArray(ins.toJsonObject())); + new Post(prop).post(Article.toJsonArray(ins.toJsonObject())); } catch (Exception e) { fail(); @@ -28,9 +38,9 @@ @SuppressWarnings("UseSpecificCatch") public void test2014_08_008473() { try { - Article ins = new Article(MonthlyIndex.MONTHLY_INDEX, "2014-August", "008473"); + Article ins = new Article(prop.getProperty("MONTHLY_INDEX"), "2014-August", "008473"); ins.load(); - Post.post(Article.toJsonArray(ins.toJsonObject())); + new Post(prop).post(Article.toJsonArray(ins.toJsonObject())); } catch (Exception e) { fail(); @@ -44,9 +54,9 @@ @SuppressWarnings("UseSpecificCatch") public void test2014_08_008479() { try { - Article ins = new Article(MonthlyIndex.MONTHLY_INDEX, "2014-August", "008479"); + Article ins = new Article(prop.getProperty("MONTHLY_INDEX"), "2014-August", "008479"); ins.load(); - Post.post(Article.toJsonArray(ins.toJsonObject())); + new Post(prop).post(Article.toJsonArray(ins.toJsonObject())); } catch (Exception e) { fail(); @@ -61,9 +71,9 @@ @SuppressWarnings("UseSpecificCatch") public void test2014_08_008480() { try { - Article ins = new Article(MonthlyIndex.MONTHLY_INDEX, "2014-August", "008480"); + Article ins = new Article(prop.getProperty("MONTHLY_INDEX"), "2014-August", "008480"); ins.load(); - Post.post(Article.toJsonArray(ins.toJsonObject())); + new Post(prop).post(Article.toJsonArray(ins.toJsonObject())); } catch (Exception e) { fail(); @@ -77,9 +87,9 @@ @SuppressWarnings("UseSpecificCatch") public void test2008_03_000000() { try { - Article ins = new Article(MonthlyIndex.MONTHLY_INDEX, "2008-March", "000000"); + Article ins = new Article(prop.getProperty("MONTHLY_INDEX"), "2008-March", "000000"); ins.load(); - Post.post(Article.toJsonArray(ins.toJsonObject())); + new Post(prop).post(Article.toJsonArray(ins.toJsonObject())); } catch (Exception e) { fail(); diff --git a/test/talkcrawler/DailyIndexTest.java b/test/talkcrawler/DailyIndexTest.java index 89dcb23..958e93d 100644 --- a/test/talkcrawler/DailyIndexTest.java +++ b/test/talkcrawler/DailyIndexTest.java @@ -1,9 +1,23 @@ package talkcrawler; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; import static junit.framework.TestCase.fail; +import org.junit.Before; import org.junit.Test; public class DailyIndexTest { + TalkCrawlerProperties prop; + + @Before + public void setUp(){ + try { + prop = new TalkCrawlerProperties().load(); + } catch (IOException ex) { + Logger.getLogger(DailyIndexTest.class.getName()).log(Level.SEVERE, null, ex); + } + } /** * Test of load method, of class MonthlyIndex. @@ -13,7 +27,7 @@ @SuppressWarnings("UseSpecificCatch") public void load_201903() { try { - DailyIndex ins = new DailyIndex(MonthlyIndex.MONTHLY_INDEX, "2019-March", "date.html"); + DailyIndex ins = new DailyIndex(prop, "2019-March", "date.html"); ins.load(); } catch(Exception e) { @@ -25,7 +39,7 @@ @SuppressWarnings("UseSpecificCatch") public void load_201408() { try { - DailyIndex ins = new DailyIndex(MonthlyIndex.MONTHLY_INDEX, "2014-August", "date.html"); + DailyIndex ins = new DailyIndex(prop, "2014-August", "date.html"); ins.load(); } catch(Exception e) {