1. 1
2. 2
3. 3
4. 4
5. 5
6. 6
7. 7
8. 8
9. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
16. 16
17. 17
18. 18
19. 19
20. 20
21. 21
22. 22
23. 23
24. 24
25. 25
26. 26
27. 27
28. 28
29. 29
30. 30
31. 31
32. 32
33. 33
34. 34
35. 35
36. 36
37. 37
38. 38
39. 39
40. 40
41. 41
42. 42
43. 43
44. 44
45. 45
46. 46
47. 47
48. 48
49. 49
50. 50
51. 51
52. 52
53. 53
54. 54
55. 55
56. 56
57. 57
58. 58
59. 59
60. 60
61. 61
62. 62
63. 63
64. 64
65. 65
66. 66
67. 67
68. 68
69. 69
70. 70
71. 71
72. 72
73. 73
74. 74
75. 75
76. 76
77. 77
78. 78
79. 79
80. 80
81. 81
82. 82
83. 83
84. 84

import json
from bs4 import BeautifulSoup
import requests
import os

def scrape_ping_ze_rhyme(force_refresh=False):
    output_file = os.path.join(os.path.dirname(__file__), 'data', 'organized_ping_ze_rhyme_dict.json')

    # Check if the file exists and if force_refresh is False
    if os.path.exists(output_file) and not force_refresh:
        print(f"JSON file already exists at {output_file}. Use `force_refresh=True` to regenerate.")
        return
    
    # Load the page
    url = 'https://zh.wikisource.org/wiki/%E5%B9%B3%E6%B0%B4%E9%9F%BB'
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')

    # Initialize an empty hash map (dictionary)
    rhyme_dict = {}
    current_section_title = None

    # Locate the main content where rhyme data is present
    content = soup.find('div', class_='mw-parser-output')

    # Iterate through all <p> tags that contain the rhyme data
    for paragraph in content.find_all('p'):
        text = paragraph.get_text(strip=True)

        # Check if the paragraph contains a rhyme section title (e.g., 上平聲一東)
        if text.startswith('上平聲') or text.startswith('下平聲') or text.startswith('上聲') or text.startswith('去聲') or text.startswith('入聲'):
            current_section_title = text.strip()
            rhyme_dict[current_section_title] = []  # Initialize an empty list for this section
        elif current_section_title:
            if '【詞】' in text or '【辭】' in text:
                text = text.replace('【詞】', '').replace('【辭】', '').strip()
            words = text.split()
            rhyme_dict[current_section_title].extend(words)
        elif text.startswith('【詞】') or text.startswith('【辭】'):
            text = text.replace('【詞】', '').replace('【辭】', '').strip()
            words = text.split()
            rhyme_dict[current_section_title].extend(words)

    def collapse_strings_in_dict(d):
        for key, value in d.items():
            if isinstance(value, list):
                d[key] = [''.join(value)  ]
            elif isinstance(value, dict):
                collapse_strings_in_dict(value) 
    collapse_strings_in_dict(rhyme_dict)

    organized_rhyme_dict = {
        "ping": {
            "上平聲部": {},
            "下平聲部": {}
        },
        "ze": {
            "上聲部": {},
            "去聲部": {},
            "入聲部": {}
        }
    }

    # Organize into ping and ze categories based on section names
    for section, words in rhyme_dict.items():
        if section.startswith("上平聲"):
            organized_rhyme_dict["ping"]["上平聲部"][section] = words
        elif section.startswith("下平聲"):
            organized_rhyme_dict["ping"]["下平聲部"][section] = words
        elif section.startswith("上聲"):
            organized_rhyme_dict["ze"]["上聲部"][section] = words
        elif section.startswith("去聲"):
            organized_rhyme_dict["ze"]["去聲部"][section] = words
        elif section.startswith("入聲"):
            organized_rhyme_dict["ze"]["入聲部"][section] = words

    # Save the result as a JSON structure
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(organized_rhyme_dict, f, ensure_ascii=False, indent=4)

    print(f"Rhyme dictionary successfully scraped and saved to {output_file}.")

if __name__ == "__main__":
    scrape_ping_ze_rhyme()