Experimental opt-in description generator

src/components/compose.css

@@ -501,6 +501,19 @@
     padding-inline: 24px;
   }
 }
+
+@keyframes breathe {
+  0% {
+    opacity: 1;
+  }
+  40% {
+    opacity: 0.4;
+  }
+  100% {
+    opacity: 1;
+  }
+}
+
 #media-sheet {
   .media-form {
     flex: 1;
@@ -514,6 +527,10 @@
       resize: none;
       width: 100%;
       /* height: 10em; */
+
+      &.loading {
+        animation: skeleton-breathe 1.5s linear infinite;
+      }
     }
 
     footer {

src/components/compose.jsx

@@ -1,6 +1,7 @@
 import './compose.css';
 
 import '@github/text-expander-element';
+import { MenuItem } from '@szhsin/react-menu';
 import equal from 'fast-deep-equal';
 import { forwardRef } from 'preact/compat';
 import { useEffect, useMemo, useRef, useState } from 'preact/hooks';
@@ -11,6 +12,7 @@ import { uid } from 'uid/single';
 import { useDebouncedCallback, useThrottledCallback } from 'use-debounce';
 import { useSnapshot } from 'valtio';
 
+import Menu2 from '../components/menu2';
 import supportedLanguages from '../data/status-supported-languages';
 import urlRegex from '../data/url-regex';
 import { api } from '../utils/api';
@@ -19,6 +21,7 @@ import emojifyText from '../utils/emojify-text';
 import localeMatch from '../utils/locale-match';
 import openCompose from '../utils/open-compose';
 import shortenNumber from '../utils/shorten-number';
+import showToast from '../utils/show-toast';
 import states, { saveStatus } from '../utils/states';
 import store from '../utils/store';
 import {
@@ -39,6 +42,8 @@ import Loader from './loader';
 import Modal from './modal';
 import Status from './status';
 
+const { PHANPY_IMG_ALT_API_URL: IMG_ALT_API_URL } = import.meta.env;
+
 const supportedLanguagesMap = supportedLanguages.reduce((acc, l) => {
   const [code, common, native] = l;
   acc[code] = {
@@ -1291,7 +1296,7 @@ const Textarea = forwardRef((props, ref) => {
   const { masto } = api();
   const [text, setText] = useState(ref.current?.value || '');
   const { maxCharacters, performSearch = () => {}, ...textareaProps } = props;
-  const snapStates = useSnapshot(states);
+  // const snapStates = useSnapshot(states);
   // const charCount = snapStates.composerCharacterCount;
 
   const customEmojis = useRef();
@@ -1645,6 +1650,7 @@ function MediaAttachment({
   onDescriptionChange = () => {},
   onRemove = () => {},
 }) {
+  const [uiState, setUIState] = useState('default');
   const supportsEdit = supports('@mastodon/edit-media-attributes');
   const { type, id, file } = attachment;
   const url = useMemo(
@@ -1653,7 +1659,7 @@ function MediaAttachment({
   );
   console.log({ attachment });
   const [description, setDescription] = useState(attachment.description);
-  const suffixType = type.split('/')[0];
+  const [suffixType, subtype] = type.split('/');
   const debouncedOnDescriptionChange = useDebouncedCallback(
     onDescriptionChange,
     250,
@@ -1699,7 +1705,8 @@ function MediaAttachment({
           autoCorrect="on"
           spellCheck="true"
           dir="auto"
-          disabled={disabled}
+          disabled={disabled || uiState === 'loading'}
+          class={uiState === 'loading' ? 'loading' : ''}
           maxlength="1500" // Not unicode-aware :(
           // TODO: Un-hard-code this maxlength, ref: https://github.com/mastodon/mastodon/blob/b59fb28e90bc21d6fd1a6bafd13cfbd81ab5be54/app/models/media_attachment.rb#L39
           onInput={(e) => {
@@ -1712,6 +1719,13 @@ function MediaAttachment({
     </>
   );
 
+  const toastRef = useRef(null);
+  useEffect(() => {
+    return () => {
+      toastRef.current?.hideToast?.();
+    };
+  }, []);
+
   return (
     <>
       <div class="media-attachment">
@@ -1785,12 +1799,68 @@ function MediaAttachment({
               <div class="media-form">
                 {descTextarea}
                 <footer>
+                  {suffixType === 'image' &&
+                    /^(png|jpe?g|gif|webp)$/i.test(subtype) &&
+                    !!states.settings.mediaAltGenerator &&
+                    !!IMG_ALT_API_URL && (
+                      <Menu2
+                        portal={{
+                          target: document.body,
+                        }}
+                        containerProps={{
+                          style: {
+                            zIndex: 1001,
+                          },
+                        }}
+                        align="center"
+                        position="anchor"
+                        overflow="auto"
+                        menuButton={
+                          <button type="button" title="More" class="plain">
+                            <Icon icon="more" size="l" alt="More" />
+                          </button>
+                        }
+                      >
+                        <MenuItem
+                          disabled={uiState === 'loading'}
+                          onClick={() => {
+                            setUIState('loading');
+                            toastRef.current = showToast({
+                              text: 'Generating description. Please wait...',
+                              duration: -1,
+                            });
+                            // POST with multipart
+                            (async function () {
+                              try {
+                                const body = new FormData();
+                                body.append('image', file);
+                                const response = await fetch(IMG_ALT_API_URL, {
+                                  method: 'POST',
+                                  body,
+                                }).then((r) => r.json());
+                                setDescription(response.description);
+                              } catch (e) {
+                                console.error(e);
+                                showToast('Failed to generate description');
+                              } finally {
+                                setUIState('default');
+                                toastRef.current?.hideToast?.();
+                              }
+                            })();
+                          }}
+                        >
+                          <Icon icon="sparkles2" />
+                          <span>Generate description…</span>
+                        </MenuItem>
+                      </Menu2>
+                    )}
                   <button
                     type="button"
                     class="light block"
                     onClick={() => {
                       setShowModal(false);
                     }}
+                    disabled={uiState === 'loading'}
                   >
                     Done
                   </button>

src/components/icon.jsx

@@ -69,6 +69,7 @@ export const ICONS = {
   history: () => import('@iconify-icons/mingcute/history-line'),
   share: () => import('@iconify-icons/mingcute/share-2-line'),
   sparkles: () => import('@iconify-icons/mingcute/sparkles-line'),
+  sparkles2: () => import('@iconify-icons/mingcute/sparkles-2-line'),
   exit: () => import('@iconify-icons/mingcute/exit-line'),
   translate: () => import('@iconify-icons/mingcute/translate-line'),
   play: () => import('@iconify-icons/mingcute/play-fill'),

src/components/media-modal.jsx

@@ -1,4 +1,4 @@
-import { Menu } from '@szhsin/react-menu';
+import { MenuDivider, MenuItem } from '@szhsin/react-menu';
 import { getBlurHashAverageColor } from 'fast-blurhash';
 import {
   useEffect,
@@ -10,6 +10,7 @@ import {
 import { useHotkeys } from 'react-hotkeys-hook';
 
 import { oklab2rgb, rgb2oklab } from '../utils/color-utils';
+import showToast from '../utils/show-toast';
 import states from '../utils/states';
 
 import Icon from './icon';
@@ -18,6 +19,8 @@ import Media from './media';
 import Menu2 from './menu2';
 import MenuLink from './menu-link';
 
+const { PHANPY_IMG_ALT_API_URL: IMG_ALT_API_URL } = import.meta.env;
+
 function MediaModal({
   mediaAttachments,
   statusID,
@@ -26,6 +29,7 @@ function MediaModal({
   index = 0,
   onClose = () => {},
 }) {
+  const [uiState, setUIState] = useState('default');
   const carouselRef = useRef(null);
 
   const [currentIndex, setCurrentIndex] = useState(index);
@@ -144,6 +148,13 @@ function MediaModal({
     );
   }, [mediaAccentColors]);
 
+  let toastRef = useRef(null);
+  useEffect(() => {
+    return () => {
+      toastRef.current?.hideToast?.();
+    };
+  }, []);
+
   return (
     <div
       class={`media-modal-container media-modal-count-${mediaAttachments?.length}`}
@@ -284,6 +295,47 @@ function MediaModal({
               <Icon icon="popout" />
               <span>Open original media</span>
             </MenuLink>
+            {import.meta.env.DEV && // Only dev for now
+              !!states.settings.mediaAltGenerator &&
+              !!IMG_ALT_API_URL &&
+              !!mediaAttachments[currentIndex]?.url &&
+              !mediaAttachments[currentIndex]?.description &&
+              mediaAttachments[currentIndex]?.type === 'image' && (
+                <>
+                  <MenuDivider />
+                  <MenuItem
+                    disabled={uiState === 'loading'}
+                    onClick={() => {
+                      setUIState('loading');
+                      toastRef.current = showToast({
+                        text: 'Attempting to describe image. Please wait...',
+                        duration: -1,
+                      });
+                      (async function () {
+                        try {
+                          const response = await fetch(
+                            `${IMG_ALT_API_URL}?image=${encodeURIComponent(
+                              mediaAttachments[currentIndex]?.url,
+                            )}`,
+                          ).then((r) => r.json());
+                          states.showMediaAlt = {
+                            alt: response.description,
+                          };
+                        } catch (e) {
+                          console.error(e);
+                          showToast('Failed to describe image');
+                        } finally {
+                          setUIState('default');
+                          toastRef.current?.hideToast?.();
+                        }
+                      })();
+                    }}
+                  >
+                    <Icon icon="sparkles2" />
+                    <span>Describe image…</span>
+                  </MenuItem>
+                </>
+              )}
           </Menu2>{' '}
           <Link
             to={`${instance ? `/${instance}` : ''}/s/${statusID}${

src/index.css

@@ -338,6 +338,9 @@ button.large {
   font-size: 125%;
   padding: 12px;
 }
+textarea:disabled {
+  background-color: var(--bg-faded-color);
+}
 
 :is(input[type='text'], textarea, select).block {
   display: block;

src/pages/settings.jsx

@@ -27,6 +27,7 @@ const TEXT_SIZES = [15, 16, 17, 18, 19, 20];
 const {
   PHANPY_WEBSITE: WEBSITE,
   PHANPY_PRIVACY_POLICY_URL: PRIVACY_POLICY_URL,
+  PHANPY_IMG_ALT_API_URL: IMG_ALT_API_URL,
 } = import.meta.env;
 
 function Settings({ onClose }) {
@@ -432,6 +433,34 @@ function Settings({ onClose }) {
                 </div>
               </div>
             </li>
+            {!!IMG_ALT_API_URL && (
+              <li>
+                <label>
+                  <input
+                    type="checkbox"
+                    checked={snapStates.settings.mediaAltGenerator}
+                    onChange={(e) => {
+                      states.settings.mediaAltGenerator = e.target.checked;
+                    }}
+                  />{' '}
+                  Image description generator{' '}
+                  <Icon icon="sparkles2" class="more-insignificant" />
+                </label>
+                <div class="sub-section insignificant">
+                  <small>
+                    Note: This feature uses external AI service, powered by{' '}
+                    <a
+                      href="https://github.com/cheeaun/img-alt-api"
+                      target="_blank"
+                      rel="noopener noreferrer"
+                    >
+                      img-alt-api
+                    </a>
+                    . May not work well. Only for images and in English.
+                  </small>
+                </div>
+              </li>
+            )}
             <li>
               <label>
                 <input

src/utils/show-toast.js

@@ -23,6 +23,7 @@ function showToast(props) {
   } else {
     toast.showToast();
   }
+  return toast;
 }
 
 export default showToast;

src/utils/states.js

@@ -59,6 +59,7 @@ const states = proxy({
     contentTranslationTargetLanguage: null,
     contentTranslationHideLanguages: [],
     contentTranslationAutoInline: false,
+    mediaAltGenerator: false,
     cloakMode: false,
   },
 });
@@ -87,6 +88,8 @@ export function initStates() {
     store.account.get('settings-contentTranslationHideLanguages') || [];
   states.settings.contentTranslationAutoInline =
     store.account.get('settings-contentTranslationAutoInline') ?? false;
+  states.settings.mediaAltGenerator =
+    store.account.get('settings-mediaAltGenerator') ?? false;
   states.settings.cloakMode = store.account.get('settings-cloakMode') ?? false;
 }
 
@@ -122,6 +125,9 @@ subscribe(states, (changes) => {
         states.settings.contentTranslationHideLanguages,
       );
     }
+    if (path.join('.') === 'settings.mediaAltGenerator') {
+      store.account.set('settings-mediaAltGenerator', !!value);
+    }
     if (path?.[0] === 'shortcuts') {
       store.account.set('shortcuts', states.shortcuts);
     }