musen_denshinno_kinjo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "musen_denshinno_kinjo.ipynb ",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPlGmUre4TcIQHgcA9QkCJM",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/disneyono/oono/blob/main/musen_denshinno_kinjo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q-Rc6axsk0zk",
        "outputId": "b6cbaa28-30b2-4919-d085-c2c836cc44ae"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: janome in /usr/local/lib/python3.7/dist-packages (0.4.2)\n",
            "もの(16) 波(15) 電波(15) これ(11) の(11) 事(10) 器(10) 受信(8) アンテナ(8) 用(6) 線(6) 電話(6) よう(5) 白金(5) 電流(5) 年(5) 日(5) 無線(4) 電信(4) 傍点(4) 発信(4) 所(4) 電気(4) 火花(4) 時(4) 通信(4) 従来(4) 式(4) 金属(4) 間(4) マルコニ(4) 十(4) 筒(4) 月(4) 寺田(3) 寅彦(3) 中(3) 入力(3) 一つ(3) 波動(3) 板(3) 利用(3) 物(3) 一(3) 二(3) 三(3) 四(3) 符号(3) 電路(3) 接続(3) "
          ]
        }
      ],
      "source": [
        "!pip install janome\n",
        "\n",
        "from janome.tokenizer import Tokenizer\n",
        "import zipfile\n",
        "import os.path, urllib.request as req\n",
        "\n",
        "# ZIPファイルをダウンロード --- (※1)\n",
        "url = \"http://www.aozora.gr.jp/cards/000042/files/61019_ruby_74765.zip\"\n",
        "local = \"61019_ruby_74765.zip\"\n",
        "if not os.path.exists(local):\n",
        "    print(\"ZIPファイルをダウンロード\")\n",
        "    req.urlretrieve(url, local)\n",
        "\n",
        "# ZIPファイル内のテキストファイルを読む --- (※2)\n",
        "zf = zipfile.ZipFile(local, 'r') # zipファイルを読む\n",
        "fp= zf.open('musen_denshinno_kinjo.txt', 'r') # アーカイブ内のテキストを読む\n",
        "bindata = fp.read()\n",
        "txt = bindata.decode('shift_jis') # テキストがShift_JISなのでデコード\n",
        "\n",
        "# 形態素解析オブジェクトの生成 --- (※3)\n",
        "t = Tokenizer()\n",
        "\n",
        "# テキストを一行ずつ処理 --- (※4)\n",
        "word_dic = {}\n",
        "lines = txt.split(\"\\r\\n\")\n",
        "for line in lines:\n",
        "    malist = t.tokenize(line)\n",
        "    for w in malist:\n",
        "        word = w.surface\n",
        "        ps = w.part_of_speech # 品詞\n",
        "        if ps.find('名詞') < 0: continue # 名詞だけカウント --- (※5)\n",
        "        if not word in word_dic:\n",
        "            word_dic[word] = 0\n",
        "        word_dic[word] += 1 # カウント\n",
        "\n",
        "# よく使われる単語を表示 --- (※6)\n",
        "keys = sorted(word_dic.items(), key=lambda x:x[1], reverse=True)\n",
        "for word,cnt in keys[:50]:\n",
        "    print(\"{0}({1}) \".format(word,cnt), end=\"\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "%cd gdrive/MyDrive/検索技術/\n",
        "!pip install gensim\n",
        "from janome.tokenizer import Tokenizer\n",
        "from gensim.models import word2vec\n",
        "import re\n",
        "\n",
        "# テキストファイルの読み込み --- (※1)\n",
        "bindata = open('musen_denshinno_kinjo.txt', 'rb').read()\n",
        "text = bindata.decode('shift_jis')\n",
        "\n",
        "# テキストの先頭にあるヘッダとフッタを削除 --- (※2)\n",
        "text = re.split(r'\\-{5,}',text)[2]\n",
        "text = re.split(r'底本：', text)[0]\n",
        "text = text.strip()\n",
        "\n",
        "# 形態素解析 --- (※3)\n",
        "t = Tokenizer()\n",
        "results = []\n",
        "# テキストを一行ずつ処理する\n",
        "lines = text.split(\"\\r\\n\")\n",
        "for line in lines:\n",
        "    s = line\n",
        "    s = s.replace('｜', '')\n",
        "    s = re.sub(r'《.+?》', '', s) # ルビを削除\n",
        "    s = re.sub(r'［＃.+?］', '', s) # 入力注を削除\n",
        "    tokens = t.tokenize(s) # 形態素解析\n",
        "    # 必要な語句だけを対象とする --- (※4)\n",
        "    r = []\n",
        "    for tok in tokens:\n",
        "        if tok.base_form == \"*\": # 単語の基本系を採用\n",
        "            w = tok.surface\n",
        "        else:\n",
        "            w = tok.base_form\n",
        "        ps = tok.part_of_speech # 品詞情報\n",
        "        hinsi = ps.split(',')[0]\n",
        "        if hinsi in ['名詞', '形容詞', '動詞', '記号']:\n",
        "            r.append(w)\n",
        "    rl = (\" \".join(r)).strip()\n",
        "    results.append(rl)\n",
        "    #print(rl) # --- 画面に分かち書きした行を表示\n",
        "\n",
        "# 書き込み先テキストを開く --- (※5)\n",
        "wakati_file = 'musen_denshinno_kinjo.wakati'\n",
        "with open(wakati_file, 'w', encoding='utf-8') as fp:\n",
        "    fp.write(\"\\n\".join(results))\n",
        "\n",
        "# Word2Vecでモデルを作成 --- (※6)\n",
        "data = word2vec.LineSentence(wakati_file)\n",
        "model = word2vec.Word2Vec(data, \n",
        "    size=200, window=10, hs=1, min_count=2, sg=1)\n",
        "model.save('musen_denshinno_kinjo.model')\n",
        "print('ok')\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sTYh787flp-M",
        "outputId": "1df19d9e-817c-46b9-c716-390b34441fdd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n",
            "/content/gdrive/MyDrive/検索技術\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.7/dist-packages (3.6.0)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.21.6)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (5.2.1)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.15.0)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.7.3)\n",
            "ok\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.models import word2vec\n",
        "\n",
        "\n",
        "model = word2vec.Word2Vec.load('musen_denshinno_kinjo.model')\n",
        "\n",
        "for word in ['電流']:\n",
        "    words = model.most_similar(positive=[word])\n",
        "    n = [w[0] for w in words]\n",
        "    print(word,\"=\", \",\".join(n))\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YAkTIwwfnxc4",
        "outputId": "623065fb-880a-4c0f-a3cb-14eece0d7649"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "電流 = 線,交番,、,電波,。,これ,する,電話,もの,器\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
            "  import sys\n"
          ]
        }
      ]
    }
  ]
}