RE: org-babel-execute-src-block filters characters from :session *shell* output

[Ihor Radchenko <yantar92@posteo.net>]

[-- Attachment #1: Type: text/plain, Size: 692 bytes --]

Ihor Radchenko <yantar92@posteo.net> writes:

> Again, we know this problem, and we normally force the prompt to be
> something more unique when creating sessions. But we do not touch the
> prompt when you point Org to an existing interactive shell buffer - that
> could be a surprise if Org mode changes the prompts by force in the
> shells you use interactively.

... on the other hand, Org mode simply hanging or emitting garbage is
also not great. So, maybe changing the prompt by force is good?

I am attaching tentative series of patches that will do exactly this -
setup prompt even in the existing shell buffers. I also changed the
babel prompt to be (1) shorter; (2) more "Babel"y.


[-- Attachment #2: 0001-Rename-org-babel-comint-prompt-regexp-old-to-.-fallb.patch --]
[-- Type: text/x-patch, Size: 7003 bytes --]

From aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6 Mon Sep 17 00:00:00 2001
Message-ID: <aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6.1719774270.git.yantar92@posteo.net>
From: Ihor Radchenko <yantar92@posteo.net>
Date: Sun, 30 Jun 2024 20:36:25 +0200
Subject: [PATCH 1/4] Rename `org-babel-comint-prompt-regexp-old' to
 `...-fallback'

* lisp/ob-comint.el (org-babel-comint-prompt-regexp-fallback): Rename
to more accurate name.  Leave the old name as an alias.
(org-babel-comint--set-fallback-prompt): Use the new name.
(org-babel-comint-with-output):
(org-babel-comint-wait-for-output):
* lisp/ob-clojure.el (ob-clojure-eval-with-inf-clojure):
* lisp/ob-haskell.el (org-babel-interpret-haskell):
* lisp/ob-ruby.el (org-babel-ruby-initiate-session):
* lisp/ob-shell.el (org-babel-sh-initiate-session):
---
 lisp/ob-clojure.el |  4 ++--
 lisp/ob-comint.el  | 19 ++++++++++---------
 lisp/ob-haskell.el |  2 +-
 lisp/ob-ruby.el    |  2 +-
 lisp/ob-shell.el   |  2 +-
 5 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/lisp/ob-clojure.el b/lisp/ob-clojure.el
index c7ebbbb95..98a66d1ec 100644
--- a/lisp/ob-clojure.el
+++ b/lisp/ob-clojure.el
@@ -237,8 +237,8 @@ (defun ob-clojure-eval-with-inf-clojure (expanded params)
 			     "clojure" (format "clojure -A%s" alias)
 			     cmd0)
 		    cmd0)))
-	(setq
-         org-babel-comint-prompt-regexp-old comint-prompt-regexp
+	(setq-local
+         org-babel-comint-prompt-regexp-fallback comint-prompt-regexp
          comint-prompt-regexp inf-clojure-comint-prompt-regexp)
 	(funcall-interactively #'inf-clojure cmd)
 	(goto-char (point-max))))
diff --git a/lisp/ob-comint.el b/lisp/ob-comint.el
index 764927af7..4d4c90c5a 100644
--- a/lisp/ob-comint.el
+++ b/lisp/ob-comint.el
@@ -58,8 +58,9 @@ (defmacro org-babel-comint-in-buffer (buffer &rest body)
 	   (let ((comint-input-filter (lambda (_input) nil)))
 	     ,@body))))))
 
-(defvar-local org-babel-comint-prompt-regexp-old nil
+(defvar org-babel-comint-prompt-regexp-fallback nil
   "Fallback regexp used to detect prompt.")
+(defvaralias 'org-babel-comint-prompt-regexp-old 'org-babel-comint-prompt-regexp-fallback)
 
 (defcustom org-babel-comint-fallback-regexp-threshold 5.0
   "Waiting time until trying to use fallback regexp to detect prompt.
@@ -69,11 +70,11 @@ (defcustom org-babel-comint-fallback-regexp-threshold 5.0
   :package-version '(Org . "9.7"))
 
 (defun org-babel-comint--set-fallback-prompt ()
-  "Swap `comint-prompt-regexp' and `org-babel-comint-prompt-regexp-old'."
-  (when org-babel-comint-prompt-regexp-old
+  "Swap `comint-prompt-regexp' and `org-babel-comint-prompt-regexp-fallback'."
+  (when org-babel-comint-prompt-regexp-fallback
     (let ((tmp comint-prompt-regexp))
-      (setq comint-prompt-regexp org-babel-comint-prompt-regexp-old
-            org-babel-comint-prompt-regexp-old tmp))))
+      (setq comint-prompt-regexp org-babel-comint-prompt-regexp-fallback
+            org-babel-comint-prompt-regexp-fallback tmp))))
 
 (defun org-babel-comint--prompt-filter (string &optional prompt-regexp)
   "Remove PROMPT-REGEXP from STRING.
@@ -144,7 +145,7 @@ (defmacro org-babel-comint-with-output (meta &rest body)
 	     (accept-process-output
               (get-buffer-process (current-buffer))
               org-babel-comint-fallback-regexp-threshold)
-             (when (and org-babel-comint-prompt-regexp-old
+             (when (and org-babel-comint-prompt-regexp-fallback
                         (> (float-time (time-since start-time))
                            org-babel-comint-fallback-regexp-threshold)
                         (progn
@@ -154,7 +155,7 @@ (defmacro org-babel-comint-with-output (meta &rest body)
                              (re-search-forward
 			      (regexp-quote ,eoe-indicator) nil t)
 			     (re-search-forward
-                              org-babel-comint-prompt-regexp-old nil t)))))
+                              org-babel-comint-prompt-regexp-fallback nil t)))))
                (org-babel-comint--set-fallback-prompt))))
 	 ;; replace cut dangling text
 	 (goto-char (process-mark (get-buffer-process (current-buffer))))
@@ -189,14 +190,14 @@ (defun org-babel-comint-wait-for-output (buffer)
         (accept-process-output
          (get-buffer-process buffer)
          org-babel-comint-fallback-regexp-threshold)
-        (when (and org-babel-comint-prompt-regexp-old
+        (when (and org-babel-comint-prompt-regexp-fallback
                    (> (float-time (time-since start-time))
                       org-babel-comint-fallback-regexp-threshold)
                    (progn
 		     (goto-char comint-last-input-end)
 		     (save-excursion
 		       (re-search-forward
-                        org-babel-comint-prompt-regexp-old nil t))))
+                        org-babel-comint-prompt-regexp-fallback nil t))))
           (org-babel-comint--set-fallback-prompt))))))
 
 (defun org-babel-comint-eval-invisibly-and-wait-for-file
diff --git a/lisp/ob-haskell.el b/lisp/ob-haskell.el
index 05f340fa0..a96a28b6b 100644
--- a/lisp/ob-haskell.el
+++ b/lisp/ob-haskell.el
@@ -153,7 +153,7 @@ (defun org-babel-interpret-haskell (body params)
   (add-hook 'inferior-haskell-hook
             (lambda ()
               (setq-local
-               org-babel-comint-prompt-regexp-old comint-prompt-regexp
+               org-babel-comint-prompt-regexp-fallback comint-prompt-regexp
                comint-prompt-regexp
                (concat haskell-prompt-regexp "\\|^λ?> "))))
   (org-babel-haskell-with-session session params
diff --git a/lisp/ob-ruby.el b/lisp/ob-ruby.el
index d920fb585..24b31c18c 100644
--- a/lisp/ob-ruby.el
+++ b/lisp/ob-ruby.el
@@ -192,7 +192,7 @@ (defun org-babel-ruby-initiate-session (&optional session params)
             (when new-session?
               (with-current-buffer session-buffer
                 (setq-local
-                 org-babel-comint-prompt-regexp-old comint-prompt-regexp
+                 org-babel-comint-prompt-regexp-fallback comint-prompt-regexp
                  comint-prompt-regexp (concat "^" org-babel-ruby-prompt))
                 (insert org-babel-ruby-define-prompt ";")
                 (insert "_org_prompt_mode=conf.prompt_mode;conf.prompt_mode=:CUSTOM;")
diff --git a/lisp/ob-shell.el b/lisp/ob-shell.el
index 32b029044..9222379a3 100644
--- a/lisp/ob-shell.el
+++ b/lisp/ob-shell.el
@@ -294,7 +294,7 @@ (defun org-babel-sh-initiate-session (&optional session _params)
                   (alist-get t org-babel-shell-set-prompt-commands))
               org-babel-sh-prompt))
             (setq-local
-             org-babel-comint-prompt-regexp-old comint-prompt-regexp
+             org-babel-comint-prompt-regexp-fallback comint-prompt-regexp
              comint-prompt-regexp
              (concat "^" (regexp-quote org-babel-sh-prompt)
                      " *"))
-- 
2.45.2


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #3: 0002-ob-shell-Arrange-unique-prompt-in-manually-started-s.patch --]
[-- Type: text/x-patch, Size: 2840 bytes --]

From c950bb79ad515e53bdac3d79da7b4352aa264e8a Mon Sep 17 00:00:00 2001
Message-ID: <c950bb79ad515e53bdac3d79da7b4352aa264e8a.1719774270.git.yantar92@posteo.net>
In-Reply-To: <aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6.1719774270.git.yantar92@posteo.net>
References: <aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6.1719774270.git.yantar92@posteo.net>
From: Ihor Radchenko <yantar92@posteo.net>
Date: Sun, 30 Jun 2024 20:37:36 +0200
Subject: [PATCH 2/4] ob-shell: Arrange unique prompt in manually started shell
 buffers

* lisp/ob-shell.el (org-babel-sh--prompt-initialized): New variable
flag that indicates whether ob-shell changed the prompt in current
comint buffer.
(org-babel-sh-initiate-session): Set unique prompt in existing
sessions if it is not yet done.

Link: https://orgmode.org/list/87o782gx7o.fsf@localhost
---
 lisp/ob-shell.el | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/lisp/ob-shell.el b/lisp/ob-shell.el
index 9222379a3..7b0d6ddab 100644
--- a/lisp/ob-shell.el
+++ b/lisp/ob-shell.el
@@ -276,16 +276,25 @@ (defvar org-babel-sh-eoe-output "org_babel_sh_eoe"
 (defvar org-babel-sh-prompt "org_babel_sh_prompt> "
   "String to set prompt in session shell.")
 
+(defvar-local org-babel-sh--prompt-initialized nil
+  "When non-nil, ob-shell already initialized the prompt in current buffer.")
+
 (defalias 'org-babel-shell-initiate-session #'org-babel-sh-initiate-session)
 (defun org-babel-sh-initiate-session (&optional session _params)
   "Initiate a session named SESSION according to PARAMS."
   (when (and session (not (string= session "none")))
     (save-window-excursion
-      (or (org-babel-comint-buffer-livep session)
+      (or (and (org-babel-comint-buffer-livep session)
+               (buffer-local-value
+                'org-babel-sh--prompt-initialized
+                (get-buffer session))
+               session)
           (progn
-	    (shell session)
-            ;; Set unique prompt for easier analysis of the output.
-            (org-babel-comint-wait-for-output (current-buffer))
+            (if (org-babel-comint-buffer-livep session)
+                (set-buffer session)
+	      (shell session)
+              ;; Set unique prompt for easier analysis of the output.
+              (org-babel-comint-wait-for-output (current-buffer)))
             (org-babel-comint-input-command
              (current-buffer)
              (format
@@ -298,6 +307,7 @@ (defun org-babel-sh-initiate-session (&optional session _params)
              comint-prompt-regexp
              (concat "^" (regexp-quote org-babel-sh-prompt)
                      " *"))
+            (setq org-babel-sh--prompt-initialized t)
 	    ;; Needed for Emacs 23 since the marker is initially
 	    ;; undefined and the filter functions try to use it without
 	    ;; checking.
-- 
2.45.2


[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #4: 0003-org-babel-sh-initiate-session-Fix-setting-non-standa.patch --]
[-- Type: text/x-patch, Size: 2243 bytes --]

From 21d9bcdc22c0b601e40316dd6df7394344d4cccf Mon Sep 17 00:00:00 2001
Message-ID: <21d9bcdc22c0b601e40316dd6df7394344d4cccf.1719774270.git.yantar92@posteo.net>
In-Reply-To: <aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6.1719774270.git.yantar92@posteo.net>
References: <aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6.1719774270.git.yantar92@posteo.net>
From: Ihor Radchenko <yantar92@posteo.net>
Date: Sun, 30 Jun 2024 20:59:22 +0200
Subject: [PATCH 3/4] org-babel-sh-initiate-session: Fix setting non-standard
 prompt

* lisp/ob-shell.el (org-babel-sh-initiate-session): Set
`comint-prompt-regexp' early, _before_ evaluating prompt change.  This
way, we make sure that comint is not stuck trying to search for the
old prompt if the new prompt no longer matches `comint-prompt-regexp'.
---
 lisp/ob-shell.el | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lisp/ob-shell.el b/lisp/ob-shell.el
index 7b0d6ddab..f148fead7 100644
--- a/lisp/ob-shell.el
+++ b/lisp/ob-shell.el
@@ -295,6 +295,11 @@ (defun org-babel-sh-initiate-session (&optional session _params)
 	      (shell session)
               ;; Set unique prompt for easier analysis of the output.
               (org-babel-comint-wait-for-output (current-buffer)))
+            (setq-local
+             org-babel-comint-prompt-regexp-fallback comint-prompt-regexp
+             comint-prompt-regexp
+             (concat "^" (regexp-quote org-babel-sh-prompt)
+                     " *"))
             (org-babel-comint-input-command
              (current-buffer)
              (format
@@ -302,11 +307,6 @@ (defun org-babel-sh-initiate-session (&optional session _params)
                               org-babel-shell-set-prompt-commands))
                   (alist-get t org-babel-shell-set-prompt-commands))
               org-babel-sh-prompt))
-            (setq-local
-             org-babel-comint-prompt-regexp-fallback comint-prompt-regexp
-             comint-prompt-regexp
-             (concat "^" (regexp-quote org-babel-sh-prompt)
-                     " *"))
             (setq org-babel-sh--prompt-initialized t)
 	    ;; Needed for Emacs 23 since the marker is initially
 	    ;; undefined and the filter functions try to use it without
-- 
2.45.2


[-- Attachment #5: 0004-org-babel-sh-prompt-Use-cuneiform-break-symbol-as-un.patch --]
[-- Type: text/x-patch, Size: 1581 bytes --]

From 5b98fef45f262e46323241c0482bf81573cdb763 Mon Sep 17 00:00:00 2001
Message-ID: <5b98fef45f262e46323241c0482bf81573cdb763.1719774270.git.yantar92@posteo.net>
In-Reply-To: <aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6.1719774270.git.yantar92@posteo.net>
References: <aee2dd3f8ad0f1658ba2a222195351da9c9b4bf6.1719774270.git.yantar92@posteo.net>
From: Ihor Radchenko <yantar92@posteo.net>
Date: Sun, 30 Jun 2024 21:00:17 +0200
Subject: [PATCH 4/4] =?UTF-8?q?org-babel-sh-prompt:=20Use=20cuneiform=20"b?=
 =?UTF-8?q?reak"=20=F0=92=86=B8=20symbol=20as=20unique=20prompt?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* lisp/ob-shell.el (org-babel-sh-prompt): Change the value of unique
prompt from long prompt string to a much shorter, but still unlikely
to appear in output, single-char prompt.

Cuneiform because Sumerian, Babylon, Tower ob Babel :)
---
 lisp/ob-shell.el | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lisp/ob-shell.el b/lisp/ob-shell.el
index f148fead7..05bded918 100644
--- a/lisp/ob-shell.el
+++ b/lisp/ob-shell.el
@@ -273,7 +273,7 @@ (defvar org-babel-sh-eoe-indicator "echo 'org_babel_sh_eoe'"
   "String to indicate that evaluation has completed.")
 (defvar org-babel-sh-eoe-output "org_babel_sh_eoe"
   "String to indicate that evaluation has completed.")
-(defvar org-babel-sh-prompt "org_babel_sh_prompt> "
+(defvar org-babel-sh-prompt "𒆸 "
   "String to set prompt in session shell.")
 
 (defvar-local org-babel-sh--prompt-initialized nil
-- 
2.45.2


[-- Attachment #6: Type: text/plain, Size: 224 bytes --]


-- 
Ihor Radchenko // yantar92,
Org mode contributor,
Learn more about Org mode at <https://orgmode.org/>.
Support Org development at <https://liberapay.com/org-mode>,
or support my work at <https://liberapay.com/yantar92>