iPhoneのショートカットから電測用マップを更新

処理

ショートカット

現在地から

  • 都道府県名
  • 市区町村名
  • 住所
  • 緯度
  • 経度

を取得してwebhookでスプレッドシートにデータを送信

スプレッドシート

webhookから送られてきたデータを最終行に追加

Githubにプログラム実行のwebhookを送信

Github

webhookを受信すると上記スプレッドシートCSVを取得し

直近3日間のデータのポイントを作成

kmzを更新

ショートカット

f:id:imabari_ehime:20220321001514p:plain

f:id:imabari_ehime:20220321001532p:plain

f:id:imabari_ehime:20220321001549p:plain

f:id:imabari_ehime:20220321001615p:plain

スプレッドシート

function doPost(e) {

  const ss = SpreadsheetApp.getActiveSpreadsheet();
  const sheet = ss.getSheetByName('data');

  let ary = [e.parameter.date, e.parameter.pref, e.parameter.city, e.parameter.address, e.parameter.lat, e.parameter.lng, e.parameter.enb, e.parameter.process];

  // 行の最後に値を追加
  sheet.appendRow(ary);

  // 3秒待機
  Utilities.sleep(3000);

  // Githubのアドレス
  const url = "https://api.github.com/repos/xxxxxx/yyyyyy/dispatches";

  let data = { event_type: "on-demand-form" };

  // Githubのtoken設定
  let options = {
    method: "POST",
    headers: {
      Authorization: "token xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
      "Content-Type": "application/json",
    },
    payload: JSON.stringify(data),
  };

  console.log(options);

  UrlFetchApp.fetch(url, options);

}

github.com

import datetime
import pathlib

import pandas as pd
import simplekml

JST = datetime.timezone(datetime.timedelta(hours=+9))

dt_now = datetime.datetime.now(JST).replace(tzinfo=None)
dt_3dy = dt_now - datetime.timedelta(days=3)

# スプレッドシートのCSVのURL
url = "https://docs.google.com/spreadsheets/d/e/2PACX-1vSroTLHVCV2xgBucqPyevEtUblVM2cIpJv6SeZTHcbu_GSFQSNUb6KQyc6CDsFWjk5gieDmx126lWRm/pub?gid=0&single=true&output=csv"

df0 = pd.read_csv(url, parse_dates=["日付"])

# eNB-LCIDが重複は一番最後のデータを反映
df0.drop_duplicates(subset=["eNB-LCID"], keep="last", inplace=True)

df1 = df0[(df0["日付"] > dt_3dy) & (df0["処理"] != "削除")].drop(["日付", "処理"], axis=1)

kml = simplekml.Kml(name="temp")

# 開局

temp_img = kml.addfile("temp.png")

# スタイル
temp_normal = simplekml.Style()
temp_normal.iconstyle.scale = 1
temp_normal.iconstyle.icon.href = temp_img

# スタイル
temp_highlight = simplekml.Style()
temp_highlight.iconstyle.scale = 1
temp_highlight.iconstyle.icon.href = temp_img

temp_stylemap = simplekml.StyleMap()
temp_stylemap.normalstyle = temp_normal
temp_stylemap.highlightstyle = temp_highlight


# スタイルマップに登録

kml.document.stylemaps.append(temp_stylemap)

fol = kml.newfolder()

if len(df1) > 0:

    df1["場所"] = df1["市町村"].str.cat(df1["住所"])

    csv_path = pathlib.Path("map", "temp.csv")
    csv_path.parent.mkdir(parents=True, exist_ok=True)

    df1.to_csv(csv_path, encoding="utf_8_sig")

    for i, r in df1.iterrows():

        pnt = fol.newpoint(name=r["場所"])
        pnt.coords = [(r["経度"], r["緯度"])]

        pnt.stylemap = kml.document.stylemaps[0]
        pnt.description = f'eNB-LCID: {r["eNB-LCID"]}'

        ex_data = simplekml.ExtendedData()

        for n, v in r.items():

            ex_data.newdata(name=str(n), value=str(v))

        pnt.extendeddata = ex_data

kmz_path = pathlib.Path("map", "temp.kmz")

kml.savekmz(kmz_path)

類似度

ensekitt.hatenablog.com

zenn.dev

!pip install pdfplumber

"""再起動"""

!apt update

!wget "https://www.city.kumamoto.jp/common/UploadFileDsp.aspx?c_id=5&id=4645&sub_id=18&flid=239935" -O data.pdf

!apt install libmagickwand-dev ghostscript

# Commented out IPython magic to ensure Python compatibility.
# %%writefile /etc/ImageMagick-6/policy.xml
# <?xml version="1.0" encoding="UTF-8"?>
# <!DOCTYPE policymap>
# 
# <policymap>
# 
#   <policy domain="resource" name="memory" value="256MiB"/>
#   <policy domain="resource" name="map" value="512MiB"/>
#   <policy domain="resource" name="width" value="16KP"/>
#   <policy domain="resource" name="height" value="16KP"/>
#   <policy domain="resource" name="area" value="128MB"/>
#   <policy domain="resource" name="disk" value="1GiB"/>
# 
#   <policy domain="delegate" rights="none" pattern="URL"/>
#   <policy domain="delegate" rights="none" pattern="HTTPS"/>
#   <policy domain="delegate" rights="none" pattern="HTTP"/>
# 
#   <policy domain="path" rights="none" pattern="@*"/>
#   <policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/>
# 
#   <policy domain="coder" rights="none" pattern="PS"/>
#   <policy domain="coder" rights="none" pattern="PS2"/>
#   <policy domain="coder" rights="none" pattern="PS3"/>
#   <policy domain="coder" rights="none" pattern="EPS"/>
#   <policy domain="coder" rights="read|write" pattern="PDF" />
#   <policy domain="coder" rights="none" pattern="XPS"/>
# </policymap>
import pdfplumber
import pandas as pd

pdf = pdfplumber.open("data.pdf")

page = pdf.pages[1]

bboxs = [
    [49, 172, 418, 509],
    [455, 172, 824, 509],
    [49, 712, 418, 1049],
    [455, 712, 824, 1049],
]

n = 1

bbox = bboxs[n]

# cropでテキスト取得
crop = page.within_bbox(bbox)

# PDF確認
im = crop.to_image(resolution=300)
im

im.save("pdf.png", format="PNG")

"""# 画像認識"""

import cv2
import numpy as np

from google.colab.patches import cv2_imshow

from matplotlib import pyplot as plt

img = cv2.imread("pdf.png")

cv2_imshow(img)

# 行数
rows = 5

# 列数
cols = 6

chunks = []

for row_img in np.array_split(img, rows, axis=0):
    for chunk in np.array_split(row_img, cols, axis=1):
        chunks.append(chunk[5:-5, 5:-5, :])

print(len(chunks))

from pathlib import Path

output_dir = Path("output")
output_dir.mkdir(exist_ok=True)

for i, chunk in enumerate(chunks):
    save_path = output_dir / f"chunk_{i:02d}.png"
    cv2.imwrite(str(save_path), chunk)

!zip -r output.zip output

"""# 画像確認"""

day1, day2 = 6, 24

chunks[day1].shape

img1 = chunks[day1][:, 90: ,:]

img1g = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)

cv2_imshow(img1)

chunks[day2].shape

img2 = chunks[day2][:, 90: ,:]

img2g = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)

cv2_imshow(img2)

"""# ヒストグラム比較

+ https://ensekitt.hatenablog.com/entry/2018/07/09/200000
"""

hist_g_1 = cv2.calcHist([img1], [0], None, [256], [0, 256])

hist_g_2 = cv2.calcHist([img2], [0], None, [256], [0, 256])

comp_hist = cv2.compareHist(hist_g_1, hist_g_2, cv2.HISTCMP_CORREL)

comp_hist

def img2hist(img):

    histrgb = []

    color = ("b", "g", "r")

    for i, col in enumerate(color):
        histrgb.append(cv2.calcHist([img], [i], None, [256], [0, 256]))
        plt.plot(histrgb[i], color=col)
        plt.xlim([0, 256])
        
    plt.show()

    return histrgb

histrgb_1 = img2hist(img1)

histrgb_2 = img2hist(img2)

histarray = np.array(histrgb_1)
histvec_1 = histarray.reshape(histarray.shape[0] * histarray.shape[1], 1)

histvec_1.shape

histarray = np.array(histrgb_2)
histvec_2 = histarray.reshape(histarray.shape[0] * histarray.shape[1], 1)

histvec_2.shape

print(cv2.compareHist(histvec_1, histvec_2, 0))

"""# akaze"""

akaze = cv2.AKAZE_create()

kp1, des1 = akaze.detectAndCompute(img1, None)
kp2, des2 = akaze.detectAndCompute(img2, None)

# 特徴量のマッチングを実行
bf = cv2.BFMatcher() # 総当たりマッチング(Brute-Force Matcher)生成

# 特徴量ベクトル同士をBrute-ForceとKNN(kth-nearest neighbor)でマッチング
matches = bf.knnMatch(des1, des2, k=2)

ratio = 0.5
good = []
for m, n in matches:
    if m.distance < ratio * n.distance:
        good.append([m])

# 対応する特徴点同士を描画
img_3 = cv2.drawMatchesKnn(img1, kp1, img2, kp2, good, None, flags=2)

# 画像表示
cv2_imshow(img_3)

len(good)

"""# 類似度

https://zenn.dev/kazuhito/articles/1dc73eeb7e1297
"""

# Average Hash(入力画像の平均ハッシュ値を計算)
hash_func = cv2.img_hash.AverageHash_create()

# Block Mean Hash(ブロック平均に基づいて画像のハッシュを計算)
hash_func = cv2.img_hash.BlockMeanHash_create()

# Color Moment Hash(カラーモーメントに基づく画像ハッシュを計算)
hash_func = cv2.img_hash.ColorMomentHash_create()

# Marr Hildreth Hash(Marr-Hildreth Operator Based Hash,最も遅いが識別性が高い)
hash_func = cv2.img_hash.MarrHildrethHash_create()

# Perceptual hash(pHash)
hash_func = cv2.img_hash.PHash_create()

# Radial Variance Hash(ラドン変換に基づく画像ハッシュを計算)
hash_func = cv2.img_hash.RadialVarianceHash_create()

"""## 比較"""

hash1 = hash_func.compute(img1)
hash2 = hash_func.compute(img2)

result = hash_func.compare(hash1, hash2)
result

熊本市ごみカレンダーのPDFからCSV作成2

github.com

!pip install pdfplumber
!apt install libmagickwand-dev ghostscript

!wget "https://www.city.kumamoto.jp/common/UploadFileDsp.aspx?c_id=5&id=4638&sub_id=20&flid=239879" -O data.pdf
import io
import re

import pandas as pd
import pdfplumber


def make_cal(se0, year, n):

    n += 3

    y, m = divmod(n, 12)

    year += y
    month = m + 1

    df0 = se0.str.split(expand=True).reset_index(drop=True)
    df1 = df0[df0.isin(days + kind)].copy().dropna(how="all")

    df2 = df1.apply(lambda x: x.dropna().reset_index(drop=True), axis=1)

    s0 = df2.to_csv(index=False, header=False)
    s1 = re.sub(",(29|30|31)", r"\n\1", s0)

    df3 = (
        pd.read_csv(io.StringIO(s1), header=None, index_col=0)
        .dropna(how="all", axis=1)
        .dropna(how="all")
        .fillna("")
        .sort_index()
    )

    df4 = (
        df3[1]
        .str.cat(df3[2], sep="・")
        .str.strip("・")
        .reset_index()
        .rename(columns={0: "day", 1: "kind"})
    )

    df4["year"] = year
    df4["month"] = month

    df4["date"] = pd.to_datetime(df4[["year", "month", "day"]])

    df4.set_index("date", inplace=True)

    return df4["kind"]


tate = [0, 32, 94, 155, 216, 277, 339, 400]
yoko = [23, 90, 158, 226, 292, 360]

bboxs = [
    [18, 154, 419, 516],
    [424, 154, 825, 516],
    [18, 693, 419, 1056],
    [424, 693, 825, 1056],
]

pdf = pdfplumber.open("data.pdf")

days = list(map(str, range(1, 32)))
kind = ["燃やすごみ", "紙", "プラ容器包装", "資源物", "ペットボトル", "特定品目", "埋立ごみ"]

dfs = []
n = 0

for i in range(1, 4):

    page = pdf.pages[i]

    for bbox in bboxs:

        crop = page.within_bbox(bbox)

        vertical = list(map(lambda x: x + bbox[0], tate))
        horizontal = list(map(lambda x: x + bbox[1], yoko))

        table_settings = {
            "vertical_strategy": "explicit",
            "explicit_vertical_lines": vertical,
            "horizontal_strategy": "explicit",
            "explicit_horizontal_lines": horizontal,
        }

        se_tmp = pd.DataFrame(crop.extract_table(table_settings)).stack().str.replace("日", "日 ", regex=True)

        se = make_cal(se_tmp, 2021, n)

        dfs.append(se)
        n += 1

df0 = pd.concat(dfs)

dt_range = pd.date_range(start="2021-04-01", end="2022-03-31")

df1 = (
    df0.reindex(dt_range, fill_value="収集なし")
    .reset_index()
    .rename({"index": "収集日", "kind": "収集区分"}, axis=1)
)

df1.to_csv("kumamoto.csv", index=False)

熊本市のごみカレンダーをスクレイピング

import datetime
import time

import pandas as pd
import requests
from bs4 import BeautifulSoup
from tqdm.notebook import tqdm

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
}

url = "https://www.city.kumamoto.jp/cal_recycle/pub/default.aspx"

areas = [
    {"smst": 1, "name": "池田・黒髪(3号線の西側)・壺川・清水・城北・高平台"},
    {"smst": 2, "name": "池上・一新・春日・慶徳・五福・城西・城東・碩台・花園"},
    {"smst": 3, "name": "小島・川尻・城山・城南・白坪・高橋・中島・古町・松尾・力合・力合西"},
    {"smst": 4, "name": "託麻北・託麻西・託麻東・託麻南・長嶺"},
    {"smst": 5, "name": "秋津・桜木・月出・桜木東・東町・山ノ内"},
    {"smst": 19, "name": "麻生田・楠・黒髪(3号線の東側)・龍田・龍田西・楡木・武蔵・弓削"},
    {"smst": 7, "name": "帯山・帯山西・託麻原・西原"},
    {"smst": 8, "name": "出水・大江・向山・白川・白山・春竹・本荘"},
    {"smst": 9, "name": "出水南・田迎・田迎西・田迎南・日吉・日吉東・御幸"},
    {"smst": 10, "name": "泉ヶ丘・画図・尾ノ上・健軍・健軍東・砂取・若葉"},
    {"smst": 11, "name": "川上(改寄・大鳥居・楠野・小糸山・明徳)・北部東(梶尾・飛田・鶴羽田)"},
    {"smst": 12, "name": "西里・川上(鹿子木・西梶尾・四方寄・飛田【八原地区】)・北部東(四方寄【東葉山団地】)"},
    {"smst": 13, "name": "河内"},
    {"smst": 14, "name": "芳野"},
    {"smst": 15, "name": "飽田東・隈庄・杉上"},
    {"smst": 16, "name": "飽田西・飽田南・富合・豊田"},
    {"smst": 17, "name": "中緑・銭塘"},
    {"smst": 18, "name": "川口・奥古閑"},
]

# 西暦

year = 2022

# スクレイピング

data = []

for area in tqdm(areas):

    for month in tqdm(range(1, 13)):

        payload = {"c_id": 14, "yy": year, "mm": month, "lmst": 1, "smst": area["smst"]}

        r = requests.get(url, headers=headers, params=payload)
        r.raise_for_status()

        soup = BeautifulSoup(r.content, "html.parser")

        table = soup.select_one("table.cal")

        for tr in table.select("tr"):

            for td in tr.select("td"):

                tag = td.select_one("span > p strong")

                if tag:

                    day = int(tag.get_text(strip=True))

                    for img in td.select("img"):

                        d = {}

                        d["id"] = area["smst"]
                        d["name"] = area["name"]

                        d["date"] = datetime.datetime(year, month, day)
                        d["kind"] = img.get("alt").replace("アイコン画像", "")

                        data.append(d)

        time.sleep(3)

    time.sleep(3)

# データラングリング

df0 = pd.DataFrame(data)

df1 = df0.groupby(["id", "name", "date"])["kind"].apply("・".join).reset_index()

df1.to_csv("kumamoto.csv", encoding="utf_8_sig", index=False)

熊本市ごみカレンダーのPDFからCSV作成1

!pip install pdfplumber
!apt install libmagickwand-dev ghostscript

!wget "https://www.city.kumamoto.jp/common/UploadFileDsp.aspx?c_id=5&id=4638&sub_id=20&flid=239879" -O data.pdf
import re

import pandas as pd
import pdfplumber

tate = [0, 32, 94, 155, 216, 277, 339, 400]
yoko = [23, 90, 158, 226, 292, 360]

bboxs = [
    [18, 154, 419, 516],
    [424, 154, 825, 516],
    [18, 693, 419, 1056],
    [424, 693, 825, 1056],
]

pdf = pdfplumber.open("data.pdf")


def make_cal(se0, year, n):

    n += 3

    y, m = divmod(n, 12)

    year += y
    month = m + 1

    se1 = (
        se0.str.replace(
            "(元日|成人の日|建国記念の日|天皇誕生日|春分の日|昭和の日|憲法記念日|みどりの日|こどもの日|海の日|山の日|敬老の日|秋分の日|スポーツの日|文化の日|勤労感謝の日|振替休日)",
            "",
            regex=True,
        )
        .str.replace("特定\n品目", "")
        .str.replace("\n+", "\n")
    )

    s = "\n".join(se1.tolist())
    
    data = re.findall(
        "(\d{1,2})\s?(燃やすごみ|紙|プラ容器包装|資源物|ペットボトル|特定品目|埋立ごみ)\s?(燃やすごみ|紙|プラ容器包装|資源物|ペットボトル|特定品目|埋立ごみ)?",
        s,
    )

    df0 = pd.DataFrame(data)
    df0.set_index(0, inplace=True)

    df1 = df0[1].str.cat(df0[2], sep="・").str.strip("・").reset_index()
    df1.set_axis(["day", "kind"], axis=1, inplace=True)

    df1["day"] = df1["day"].astype(int)

    df1["year"] = year
    df1["month"] = month

    df1["date"] = pd.to_datetime(df1[["year", "month", "day"]])

    df1.set_index("date", inplace=True)

    return df1["kind"]


dfs = []
n = 0

for i in range(1, 4):

    page = pdf.pages[i]

    for bbox in bboxs:

        crop = page.within_bbox(bbox)

        im = crop.to_image()
        im.save(f"image{n}.png", format="PNG")

        vertical = list(map(lambda x: x + bbox[0], tate))
        horizontal = list(map(lambda x: x + bbox[1], yoko))

        table_settings = {
            "vertical_strategy": "explicit",
            "explicit_vertical_lines": vertical,
            "horizontal_strategy": "explicit",
            "explicit_horizontal_lines": horizontal,
        }

        se_tmp = pd.DataFrame(crop.extract_table(table_settings)).stack()

        se = make_cal(se_tmp, 2021, n)

        dfs.append(se)
        n += 1

df0 = pd.concat(dfs)

dt0 = pd.date_range(start="2021-04-01", end="2022-03-31")

df1 = (
    df0.reindex(dt0, fill_value="収集なし")
    .reset_index()
    .rename({"index": "収集日", "kind": "収集区分"}, axis=1)
)

df1.to_csv("kumamoto.csv", index=False)