From 69db5eef49c69287270b1f171e53ddb5e76625dd Mon Sep 17 00:00:00 2001 From: Bruno BARBIER Date: Fri, 16 Feb 2024 14:33:33 +0100 Subject: [PATCH 8/8] scratch/bba-ob-core-async: Some temporary test files --- scratch/bba-ob-core-async/my-async-tests.el | 172 +++++++ scratch/bba-ob-core-async/my-async-tests.org | 484 +++++++++++++++++++ 2 files changed, 656 insertions(+) create mode 100644 scratch/bba-ob-core-async/my-async-tests.el create mode 100644 scratch/bba-ob-core-async/my-async-tests.org diff --git a/scratch/bba-ob-core-async/my-async-tests.el b/scratch/bba-ob-core-async/my-async-tests.el new file mode 100644 index 000000000..3550fc88f --- /dev/null +++ b/scratch/bba-ob-core-async/my-async-tests.el @@ -0,0 +1,172 @@ +;;; my-async-tests.el --- Scratch/temporary file: some tests about async -*- lexical-binding: t -*- + +;; Copyright (C) 2024 Bruno BARBIER + +;; Author: Bruno BARBIER +;; Status: Temporary tests. +;; Compatibility: GNU Emacs 30.0.50 +;; +;; This file is NOT part of GNU Emacs. + +;; This program is free software; you can redistribute it and/or +;; modify it under the terms of the GNU General Public License as +;; published by the Free Software Foundation; either version 2 of +;; the License, or (at your option) any later version. + +;; This program is distributed in the hope that it will be +;; useful, but WITHOUT ANY WARRANTY; without even the implied +;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +;; PURPOSE. See the GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public +;; License along with this program; if not, write to the Free +;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +;; MA 02111-1307 USA + +(require 'org) +(require 'org-elib-async) + +(defun my-shell-babel-schedule (lang body params handle-feedback) + "Execute the bash script BODY. +Execute the shell script BODY using bash. Use HANDLE-FEEDBACK to report +the outcome (success or failure)." + (unless (equal "bash" lang) + (error "Only for bash")) + (funcall handle-feedback (list :pending "started")) + (org-elib-async-process (list "bash") :input body :callback handle-feedback)) + + +(defun my-org-babel-python-how-to-execute (body params) + "Return how to execute BODY using python. +Return how to execute, as expected by +`org-elib-async-comint-queue--execution'." + ;; Code mostly extracted from ob-python, following + ;; `org-babel-python-evaluate-session'. + ;; Results are expected to differ from ob-python as we follow the + ;; same process for all execution paths: asynchronous or not, with + ;; session or without. + (let* ((org-babel-python-command + (or (cdr (assq :python params)) + org-babel-python-command)) + (session-key (org-babel-python-initiate-session + (cdr (assq :session params)))) + (graphics-file (and (member "graphics" (assq :result-params params)) + (org-babel-graphical-output-file params))) + (result-params (cdr (assq :result-params params))) + (result-type (cdr (assq :result-type params))) + (results-file (when (eq 'value result-type) + (or graphics-file + (org-babel-temp-file "python-")))) + (return-val (when (eq result-type 'value) + (cdr (assq :return params)))) + (preamble (cdr (assq :preamble params))) + (full-body + (concat + (org-babel-expand-body:generic + body params + (org-babel-variable-assignments:python params)) + (when return-val + (format "\n%s" return-val)))) + (post-process + (lambda (r) + (setq r (string-trim r)) + (when (string-prefix-p "Traceback (most recent call last):" r) + (signal 'user-error (list r))) + (when (eq 'value result-type) + (setq r (org-babel-eval-read-file results-file))) + (org-babel-reassemble-table + (org-babel-result-cond result-params + r + (org-babel-python-table-or-string r)) + (org-babel-pick-name (cdr (assq :colname-names params)) + (cdr (assq :colnames params))) + (org-babel-pick-name (cdr (assq :rowname-names params)) + (cdr (assq :rownames params)))))) + (tmp-src-file (org-babel-temp-file "python-")) + (session-body + ;; The real code we evaluate in the session. + (pcase result-type + (`output + (format (string-join + (list "with open('%s') as f:\n" + " exec(compile(f.read(), f.name, 'exec'))\n")) + (org-babel-process-file-name + tmp-src-file 'noquote))) + (`value + ;; FIXME: In this case, any output is an error. + (org-babel-python-format-session-value + tmp-src-file results-file result-params)))) + comint-buffer + finally) + + + (unless session-key + ;; No session. We create a temporary one and use 'finally' to + ;; destroy it once we are done. + ;; + ;; FIXME: This session code should be refactored and moved into + ;; ob-core. + (setq session-key (org-babel-python-initiate-session + ;; We can't use a simple `generate-new-buffer' + ;; due to the earmuffs game. + (org-babel-python-without-earmuffs + (format "*ob-python-no-session-%s*" (org-id-uuid))))) + (setq finally (lambda () + (when-let ((s-buf + (get-buffer (org-babel-python-with-earmuffs session-key)))) + ;; We cannot delete it immediately as we are called from it. + (run-with-idle-timer + 0.1 nil + (lambda () + (when (buffer-live-p s-buf) + (let ((kill-buffer-query-functions nil) + (kill-buffer-hook nil)) + (kill-buffer s-buf))))))))) + + (org-elib-async-comint-queue-init-if-needed session-key) + (setq comint-buffer + (get-buffer (org-babel-python-with-earmuffs session-key))) + (with-temp-file tmp-src-file + (insert (if (and graphics-file (eq result-type 'output)) + (format org-babel-python--output-graphics-wrapper + full-body graphics-file) + full-body))) + + (lambda (&rest q) + (pcase q + (`(:instrs-to-enter) + ;; FIXME: This is wrong. + "import sys; sys.ps1=''; sys.ps2=''") + (`(:instrs-to-exit)) + (`(:finally) (when finally (funcall finally))) + (`(:instr-to-emit-tag ,tag) (format "print ('%s')" tag)) + (`(:post-process ,r) (when post-process (funcall post-process r))) + (`(:send-instrs-to-session ,code) + ;; See org-babel-python-send-string + (with-current-buffer comint-buffer + (let ((python-shell-buffer-name + (org-babel-python-without-earmuffs session-key))) + (python-shell-send-string (concat code "\n"))))) + (`(:get-code) session-body) + (`(:get-comint-buffer) comint-buffer) + (_ (error "Unknown query")))))) + + + +(defun my-org-babel-schedule (lang body params handle-feedback) + "Schedule the execution of BODY according to PARAMS. +This function is called by `org-babel-execute-src-block'. Return a +function that waits and returns the result on success, raise on failure." + (cl-assert (equal "python" lang)) + (let ((exec (my-org-babel-python-how-to-execute body params))) + (org-elib-async-comint-queue--push exec :handle-feedback handle-feedback))) + + +(defun my-org-babel-execute (lang body params) + "Execute Python BODY according to PARAMS. +This function is called by `org-babel-execute-src-block'." + ;; We just start the asynchronous execution, wait for it, and return + ;; the result (or raise the exception). No custom code, and, + ;; synchronous and asynchronous should just mix nicely together. + (cl-assert (equal "python" lang)) + (funcall (my-org-babel-schedule lang body params nil))) diff --git a/scratch/bba-ob-core-async/my-async-tests.org b/scratch/bba-ob-core-async/my-async-tests.org new file mode 100644 index 000000000..263ca77a2 --- /dev/null +++ b/scratch/bba-ob-core-async/my-async-tests.org @@ -0,0 +1,484 @@ +#+PROPERTY: HEADER-ARGS+ :eval no-export :exports both +* Intro + +An org document with code blocks to help test the proposed patches. + +You need to load: + #+begin_src elisp :results silent + (load-file "my-async-tests.el") + #+end_src + + + +Emacs and org versions: + #+begin_src elisp + (mapcar (lambda (sb) (list sb (symbol-value sb))) + '(emacs-version org-version)) + #+end_src + + #+RESULTS: + | emacs-version | 30.0.50 | + | org-version | 9.7-pre | + +Note that we've disabled eval on export: export doesn't know it needs +to wait for asynchronous results. + +* A simple bash example + :PROPERTIES: + :header-args:bash: :execute-with my-shell-babel :nasync yes + :END: + +The package `my-async-tests.el' contains the function +`my-shell-babel-schedule' to evaluate shell script asynchronously. + +The header-args properties above request asynchronous execution for +bash (:nasync yes), and, tells ob-core to use the prefix +`my-shell-babel' when looking for functions to evaluate a source +block. Thus, org will delegate execution to `my-shell-babel-schedule'. +We don't have `my-shell-babel-execute', so, in this case, :nasync must +be yes. + +A simple execution: + #+begin_src bash + date + #+end_src + + #+RESULTS: + : Fri Feb 16 18:11:08 CET 2024 + +A tricky computation takes some time: + #+begin_src bash + sleep 5; date + #+end_src + + #+RESULTS: + : Fri Feb 16 17:58:23 CET 2024 + +An example of a failure: + #+begin_src bash + sleepdd 1; false + #+end_src + + #+RESULTS: + +* Python + :PROPERTIES: + :header-args:python: :execute-with my-org-babel :nasync yes + :header-args:python+: :session py-async + :END: + +Used =header-args= properties: + - =:execute-with my-org-babel=: look for functions with the prefix `my-org-babel' to execute + blocks (for the asynchronous case use + `my-org-babel-schedule', and, for the synchronous case + `my-org-babel-execute'). These functions are defined in [[file:my-async-tests.el]]. + + - =:nasync yes=: by default, execute asynchronously (use `my-org-babel-schedule'). + + - =:session py-async= by default, use a session named "py-async". + +** basic examples +*** async with a session +A very simple test: + #+begin_src python + 2+3 + #+end_src + + #+RESULTS: + : 5 + +Let's import the module time in our session. + #+begin_src python :results silent + import time + #+end_src + + #+RESULTS: + +(Yes, =:results silent= needs some work.) + + + +A table that requires some time to compute: + #+begin_src python + start = time.time() + time.sleep(1) + end = time.time() + ["%.1fs" % t for t in [start, end, end-start]] + #+end_src + + #+RESULTS: + | 1708103472.9s | 1708103473.9s | 1.0s | + + +An error (click on the error , , to see the details): + #+begin_src python + 2/0 + #+end_src + + #+RESULTS: + + +*** async with no session + :PROPERTIES: + :header-args:python+: :session none + :END: + +A very simple test: + #+begin_src python + 2+3 + #+end_src + + #+RESULTS: + : 5 + +Let's import the module time in our session. + #+begin_src python :results silent + import time + #+end_src + + #+RESULTS: + +(Yes, =:results silent= needs some work.) + + + +A table that requires some time to compute: + #+begin_src python + start = time.time() + time.sleep(1) + end = time.time() + ["%.1fs" % t for t in [start, end, end-start]] + #+end_src + + #+RESULTS: + | 1708083470.9s | 1708083471.9s | 1.0s | + +Yes, it failed, as expected. "import time" was done in its own +temporary session. The old result is preserved; the error is display +as an overlay. Click on it to get more info about the error. + + +Let's fix it, adding the import line: + #+begin_src python + import time + start = time.time() + time.sleep(1) + end = time.time() + ["%.1fs" % t for t in [start, end, end-start]] + #+end_src + + #+RESULTS: + | 1708102948.9s | 1708102949.9s | 1.0s | + + +An error (click on the error , , to see the details): + #+begin_src python + 2/0 + #+end_src + + #+RESULTS: + + + +*** sync with a session + :PROPERTIES: + :header-args:python+: :session py-sync-session :nasync no + :END: + +A very simple test: + #+begin_src python + 2+3 + #+end_src + + #+RESULTS: + : 5 + +Let's import the module time in our session. + #+begin_src python :results silent + import time + #+end_src + +(Yes, =:results silent= needs some work.) + + + +A table that requires some time to compute: + #+begin_src python + start = time.time() + time.sleep(1) + end = time.time() + ["%.1fs" % t for t in [start, end, end-start]] + #+end_src + + #+RESULTS: + | 1708102997.5s | 1708102998.5s | 1.0s | + + + +An error (click on the error , , to see the details): + #+begin_src python + 2/0 + #+end_src + + #+RESULTS: + + +*** sync with no session + :PROPERTIES: + :header-args:python+: :session none :nasync no + :END: + +A very simple test: + #+begin_src python + 2+3 + #+end_src + + #+RESULTS: + : 5 + +Let's import the module time in our session. + #+begin_src python :results silent + import time + #+end_src + +(Yes, =:results silent= needs some work.) + + + +A table that requires some time to compute: + #+begin_src python + start = time.time() + time.sleep(1) + end = time.time() + ["%.1fs" % t for t in [start, end, end-start]] + #+end_src + + #+RESULTS: + | 1708083470.9s | 1708083471.9s | 1.0s | + +Yes, that fails (no session), displaying the details in a popup. Let's +fix it: + #+begin_src python + import time + start = time.time() + time.sleep(1) + end = time.time() + ["%.1fs" % t for t in [start, end, end-start]] + #+end_src + + #+RESULTS: + | 1708103039.0s | 1708103040.0s | 1.0s | + + + +An error (click on the error , , to see the details): + #+begin_src python + 2/0 + #+end_src + + #+RESULTS: + + +** worg examples + +Let's import matplotlib in our session. + + #+begin_src python + import matplotlib + import matplotlib.pyplot as plt + #+end_src + + #+RESULTS: + : None + +A figure in a PDF, asynchronous case. + #+begin_src python :results file link + fig=plt.figure(figsize=(3,2)) + plt.plot([1,3,2]) + fig.tight_layout() + + fname = 'myfig-async.pdf' + plt.savefig(fname) + fname # return this to org-mode + #+end_src + + #+RESULTS: + [[file:myfig-async.pdf]] + + +A figure in a PDF, synchronous case. + #+begin_src python :results file link :nasync no + fig=plt.figure(figsize=(3,2)) + plt.plot([1,3,2]) + fig.tight_layout() + + fname = 'myfig-sync.pdf' + plt.savefig(fname) + fname # return this to org-mode + #+end_src + + #+RESULTS: + [[file:myfig-sync.pdf]] + + + +A PNG figure, asynchronous case. + #+begin_src python :results graphics file output :file boxplot.png + fig=plt.figure(figsize=(3,2)) + plt.plot([1,3,2]) + fig.tight_layout() + fig + #+end_src + + #+RESULTS: + [[file:boxplot.png]] + +Same, but using the =:return= keyword. + #+begin_src python :return "plt.gcf()" :results graphics file output :file boxplot.png + fig=plt.figure(figsize=(3,2)) + plt.plot([1,3,2]) + fig.tight_layout() + #+end_src + + #+RESULTS: + [[file:boxplot.png]] + +Same, asynchronous but without a session this time. + #+begin_src python :return "plt.gcf()" :results graphics file output :file boxplot-no-sess-a-y.png :session none + import matplotlib + import matplotlib.pyplot as plt + fig=plt.figure(figsize=(3,2)) + plt.plot([1,3,2]) + fig.tight_layout() + #+end_src + + #+RESULTS: + [[file:boxplot-no-sess-a-y.png]] + + +Lists are table, + #+begin_src python + [1,2,3] + #+end_src + + #+RESULTS: + | 1 | 2 | 3 | + +unless requested otherwise. + #+begin_src python :results verbatim + [1,2,3] + #+end_src + + #+RESULTS: + : [1, 2, 3] + + +Dictionaries are tables too. + #+begin_src python :results table + {"a": 1, "b": 2} + #+end_src + + #+RESULTS: + | a | 1 | + | b | 2 | + + +Let's try the example with Panda. + #+begin_src python :results none + import pandas as pd + import numpy as np + #+end_src + + #+RESULTS: + : None + + #+begin_src python :results table + pd.DataFrame(np.array([[1,2,3],[4,5,6]]), + columns=['a','b','c']) + #+end_src + + #+RESULTS: + | | a | b | c | + |---+---+---+---| + | 0 | 1 | 2 | 3 | + | 1 | 4 | 5 | 6 | + +And the synchronous case? + + #+begin_src python :results table :nasync no + pd.DataFrame(np.array([[1,2,3],[4,5,6]]), + columns=['a','b','c']) + #+end_src + + #+RESULTS: + | | a | b | c | + |---+---+---+---| + | 0 | 1 | 2 | 3 | + | 1 | 4 | 5 | 6 | + + + +Without session ? + + #+begin_src python :results table :session none + pd.DataFrame(np.array([[1,2,3],[4,5,6]]), + columns=['a','b','c']) + #+end_src + + #+RESULTS: + | | a | b | c | + |---+---+---+---| + | 0 | 1 | 2 | 3 | + | 1 | 4 | 5 | 6 | + +Right, we need to import the libraries (no session). + + #+begin_src python :results table :session none + import pandas as pd + import numpy as np + pd.DataFrame(np.array([[1,2,3],[4,5,6]]), + columns=['a','b','c']) + #+end_src + + #+RESULTS: + | | a | b | c | + |---+---+---+---| + | 0 | 1 | 2 | 3 | + | 1 | 4 | 5 | 6 | + + +** inline examples + + A simple asynchronous inline src_python{3*2} {{{results(=6=)}}}. + + An other one containing a mistake src_python{2/0} {{{results(=6=)}}} + (click on the error to see the details). + + + Some very slow inline asynchronous computations that all run in + the same session. You need to execute the 3 of them at once. Here + is the first one src_python[:return "\"OK1\""]{import time; + time.sleep(5)} {{{results(=OK1=)}}} and a second one + src_python[:return "\"OK1 bis\""]{import time; time.sleep(5)} + {{{results(=OK1 bis=)}}} and the third one src_python[:return + "\"OK2\""]{import time; time.sleep(5)} {{{results(=OK2=)}}}. + + Yes, the previous paragraph is unreadable; it's on purpose, to + check that ob-core can figure it out. + + Let's repeat, in a more readable way, and making the last one + synchronous. + + Some very slow inline computations that all run in the same + session. Here is the first asynchronous one + src_python[:return"\"OK1\""]{import time; time.sleep(5)} {{{results(=None=)}}} + and a second one, asynchronous too: + src_python[:return "\"OK1 bis\""]{import time; time.sleep(5)} {{{results(=OK1 bis=)}}} + and finally, a third one, synchronous this one: + src_python[:nasync no :return "\"OK2\""]{import time; time.sleep(5)} {{{results(=OK2=)}}}. + + Note that, once the user executes the last synchronous block, the + user is blocked until the synchronous execution can start + (i.e. all previous asynchronous executions are done) and until + it's done. The display is updated though, to see the asynchronous + progress. -- 2.43.0