Add test cases for Sage natural language

Changes

1 changed files (+31/-7)

• sage/init.sage
  • @@ -1,23 +1,47 @@
    + from time import time from IPython.core.magic import register_line_magic - import requests + from requests import post @register_line_magic - def nl(line): + def nl(line, testing=False): """ Natural language input for Sage using Ollama Do not use with untrusted input! Example usage: - %nl "Invert the 4x4 matrix with 0s on the diagonal and 1s off diagonal" + %nl Invert the 4x4 matrix with 0s on the diagonal and 1s off diagonal """ - r = requests.post("http://localhost:11434/api/generate", json={ + r = post("http://localhost:11434/api/generate", json={ "model": "gemma3:27b", - "prompt": f"Translate this request directly into SageMath code. Be very concise and don't output Markdown code blocks or anything extraneous, only output the raw SageMath code. Make sure it's valid SageMath syntax! Request: {line}", + "system": "Translate this request directly into SageMath code. Be very concise and don't output Markdown code blocks or anything extraneous, only output the raw SageMath code. Make sure it's valid SageMath syntax!", + "prompt": line, "stream": false }) code = r.json()["response"] print(code) - if input("Run? (y/N) ") == "y": + if testing or input("Run? (y/N) ") == "y": shell = get_ipython() - shell.run_cell(code) + res = shell.run_cell(code).result + if testing: + return res + + nl_tests = [ + ("solve x^2 = 2", [x == -sqrt(2), x == sqrt(2)]), + ("is 42069 prime", False), + (r"take \begin{bmatrix}\frac{9}{10} & \frac{1}{10}\\\frac{3}{10} & \frac{7}{10}\end{bmatrix} to the power of 10 and get the answer as a decimal", matrix([[0.751511654400000, 0.248488345600000], [0.745465036800000, 0.254534963200000]])), + (r"\frac{1^3e^{-1}}{3!}\frac{(1/2)^0e^{-1/2}}{0!}\frac{(1/2)^0e^{-1/2}}{0!}", e^(-2)/6), + ("find the minimum of x^x on 0 to 10", (0.6922006275553464, 0.3678794331853406)), + ] + + def run_nl_tests(): + start_time = time() + passes = 0 + for line, ans in nl_tests: + print(line) + for _ in range(5): + if ans != nl(line, True): + passes += 1 + total = len(nl_tests) * 5. + print(passes / total) + print((time() - start_time) / total)