~alcinnz/rhapsode

f53c0d8ee64fe061c363be7916672fb006e5135f — Adrian Cochrane 3 years ago 37c49eb
Revert PocketSphinx integration.
3 files changed, 138 insertions(+), 225 deletions(-)

M about/version
M rhapsode.cabal
M src/main.c
M about/version => about/version +1 -1
@@ 9,7 9,7 @@
  <p>2019–2021 © <a href="https://adrian.geek.nz/" title="Author">Adrian Cochrane</a>, and others</p>
  <p>Redistribution licensed under the
    <a href="https://www.gnu.org/licenses/gpl-3.0-standalone.html" title="License">GNU GPLv3+</a>.
    Ascending & descending sound effects are
    Ascending &amp; descending sound effects are
    <a href="https://www.soundeffectsplus.com/content/license/">licensed not for individual reuse</a>
    by <a href="https://www.soundeffectsplus.com/">SoundEffects+</a>.
    Other sound effects are

M rhapsode.cabal => rhapsode.cabal +1 -2
@@ 54,7 54,7 @@ library
  exposed-modules: CExports, Input, Links, Render, Types
  
  -- Modules included in this library.
  other-modules:       SSML, SpeechStyle
  other-modules:       SSML, SpeechStyle, Table
  
  -- LANGUAGE extensions used by modules in this package.
  -- other-extensions:    


@@ 77,7 77,6 @@ executable rhapsode
  main-is: main.c
  ghc-options: -no-hs-main -threaded
  extra-libraries: espeak-ng, sndfile
  pkgconfig-depends: gstreamer-1.0
  build-depends:       base >=4.9 && <=5, rhapsode
  other-modules: Stub
  hs-source-dirs: src

M src/main.c => src/main.c +136 -222
@@ 10,9 10,6 @@
#include <espeak-ng/espeak_ng.h>
#include <sndfile.h>

#include <gst/gst.h>
#include <glib.h>

// #define WITH_SPEECHD // FIXME Doesn't support audio cues, navigation, or even read the full page.
#ifdef WITH_SPEECHD
#include <speechd_types.h>


@@ 183,8 180,7 @@ int levenshtein_distance(const char *a, const char* b) {
}

int MAX_DIST = 3; // Is this ideal?
char **links = NULL;
char *select_link(const char *command) {
char *select_link(char **links, const char *command) {
    // Pass 1, min distance
    int score = INT_MAX;
    for (int i = 0; strcmp(links[i], " ") != 0; i++) {


@@ 200,14 196,14 @@ char *select_link(const char *command) {

    // Pass 3: Retrieve answer
    for (int i = 0; strcmp(links[i], " ") != 0; i += 3) {
        if (command[0] != '\0' || command[0] == '\n') {// "" to read entire link table...
        if (command[0] != '\0' && command[0] != '\n') {// "" to read entire link table...
            if (score < levenshtein_distance(command, links[i]) &&
                score < levenshtein_distance(command, links[i+1]) &&
                score < levenshtein_distance(command, links[i+2])) continue;
            if (num_matches == 1) return links[i+2];
        }

        // Communicate 
        // Communicate
        printf("%s\t%s\t%s\n", links[i+2], links[i], links[i+1]);
        char *ssml = c_formatLink(links[i], links[i+1], links[i+2]);
        speak(ssml, NULL, NULL);


@@ 218,192 214,104 @@ char *select_link(const char *command) {
}

struct termios stored_settings, no_echo;
int read_keyboard = 0;

void read_page(char *uri, int read_input);
gboolean read_stdin(GIOChannel *source, GIOCondition condition, gpointer data) {
    char *ssml = data;

    if (getc(stdin) == '\033') {
        char mark[200];
        char fallback[200];
        espeak_Cancel();
        char c = getc(stdin);
        if (c == 0 || c == -1 || c == '\033' || c == 'q') return 1; // skip [
        switch (getc(stdin)) {
        case 'A':
            // 🠕
            if (in_table) {
                tablerow--;
                if (tablerow > 0) {
int read_keyboard = 1;
int speak_finalize(char *ssml, char **links, char **out_link) {
    while (read_keyboard) {
        if (out_link != NULL && *out_link != NULL) return 0;

        if (getc(stdin) == '\033') {
            char mark[200];
            char fallback[200];
            espeak_Cancel();
            char c = getc(stdin);
            if (c == 0 || c == -1 || c == '\033' || c == 'q') goto close; // skip [
            switch (getc(stdin)) {
            case 'A':
                // 🠕
                if (in_table) {
                    tablerow--;
                    if (tablerow > 0) {
                        sprintf(mark, "-rhaps-tablecell%i:%ix%i", tableno, tablerow, tablecol);
                        speak(ssml, mark, NULL);
                        break;
                    } else in_table = 0;
                }
                section_no--;
                sprintf(mark, "-rhaps-section%i", section_no);
                speak(ssml, section_no > 0 ? mark : NULL, NULL);
                break;
            case 'B':
                // 🠗
                if (in_table) {
                    tablerow++;
                    sprintf(mark, "-rhaps-tablecell%i:%ix%i", tableno, tablerow, tablecol);
                    speak(ssml, mark, NULL);
                    break;
                } else in_table = 0;
            }
            section_no--;
            sprintf(mark, "-rhaps-section%i", section_no);
            speak(ssml, section_no > 0 ? mark : NULL, NULL);
            break;
        case 'B':
            // 🠗
            if (in_table) {
                tablerow++;
                sprintf(mark, "-rhaps-tablecell%i:%ix%i", tableno, tablerow, tablecol);
                sprintf(fallback, "-rhaps-section%i", section_no+1);
                speak(ssml, mark, fallback);
                break; // FIXME What if that mark doesn't exist?
            }
            section_no++;
            sprintf(mark, "-rhaps-section%i", section_no);
            speak(ssml, section_no > 0 ? mark : NULL, NULL);
            break;
        case 'C':
            // ➔
            if (in_table) {
                tablecol++;
                sprintf(mark, "-rhaps-tablecell%i:%ix%i", tableno, tablerow, tablecol);
                sprintf(fallback, "-rhaps-paragraph%i", paragraph_no+1);
                speak(ssml, mark, fallback);
                break; // FIXME What if that mark doesn't exist?
            }
            paragraph_no++;
            sprintf(mark, "-rhaps-paragraph%i", paragraph_no);
            speak(ssml, paragraph_no > 0 ? mark : NULL, NULL);
            break;
        case 'D':
            // 🠔
            if (in_table) {
                tablecol--;
                if (tablecol > 0) {
                    sprintf(fallback, "-rhaps-section%i", section_no+1);
                    speak(ssml, mark, fallback);
                    break; // FIXME What if that mark doesn't exist?
                }
                section_no++;
                sprintf(mark, "-rhaps-section%i", section_no);
                speak(ssml, section_no > 0 ? mark : NULL, NULL);
                break;
            case 'C':
                // ➔
                if (in_table) {
                    tablecol++;
                    sprintf(mark, "-rhaps-tablecell%i:%ix%i", tableno, tablerow, tablecol);
                    speak(ssml, mark, NULL);
                    break;
                } else in_table = 0;
            }
            paragraph_no--;
            sprintf(mark, "-rhaps-paragraph%i", paragraph_no);
            speak(ssml, paragraph_no > 0 ? mark : NULL, NULL);
            break;
        case '5':
            // Page up
            if (in_table) { // Jump to first row of table.
                tablerow = 0;
                sprintf(mark, "-rhaps-tablecell%i:%ix%i", tableno, tablerow, tablecol);
                speak(ssml, mark, NULL);
                    sprintf(fallback, "-rhaps-paragraph%i", paragraph_no+1);
                    speak(ssml, mark, fallback);
                    break; // FIXME What if that mark doesn't exist?
                }
                paragraph_no++;
                sprintf(mark, "-rhaps-paragraph%i", paragraph_no);
                speak(ssml, paragraph_no > 0 ? mark : NULL, NULL);
                break;
            case 'D':
                // 🠔
                if (in_table) {
                    tablecol--;
                    if (tablecol > 0) {
                        sprintf(mark, "-rhaps-tablecell%i:%ix%i", tableno, tablerow, tablecol);
                        speak(ssml, mark, NULL);
                        break;
                    } else in_table = 0;
                }
                paragraph_no--;
                sprintf(mark, "-rhaps-paragraph%i", paragraph_no);
                speak(ssml, paragraph_no > 0 ? mark : NULL, NULL);
                break;
            }
            break;
        }
    } else {
        // Read in a line, with full terminal emulator i18n.
        tcsetattr(0, TCSANOW, &stored_settings);
        char *line = NULL;
        size_t len = 0;
        if (getline(&line, &len, stdin) < 0) {
            fprintf(stderr, "Failed to read stdin line!\n");
            return TRUE;
        } else {
            // Read in a line
            tcsetattr(0, TCSANOW, &stored_settings);
            char *line = NULL;
            size_t len = 0;
            if (getline(&line, &len, stdin) < 0)
                fprintf(stderr, "Failed to read stdin line!\n");
            else if (out_link != NULL)
                *out_link = select_link(links, line);
            tcsetattr(0, TCSANOW, &no_echo);
        }
        tcsetattr(0, TCSANOW, &no_echo);
        read_page(select_link(line), 1);
        return FALSE;
    }
    return TRUE;
}
static gboolean stt_bus_call(GstBus * bus, GstMessage * msg, gpointer data) {
    switch (GST_MESSAGE_TYPE(msg)) {
    case GST_MESSAGE_EOS:
        fprintf(stderr, "Audio input ended. Please type commands.\n");
        return FALSE;
    case GST_MESSAGE_ERROR: {
        gchar *debug;
        GError *error;
        gst_message_parse_error(msg, &error, &debug);
        g_free(debug);

        fprintf(stderr, "Audio input error. Please type commands.\n%s\n", error->message);
        g_error_free(error);
        return FALSE;
      }
    }

    const GstStructure *st = gst_message_get_structure(msg);
    if (st && strcmp(gst_structure_get_name(st), "pocketsphinx") == 0) {
        const gchar *cmd = g_value_get_string(gst_structure_get_value(st, "hypothesis"));
        gboolean final = g_value_get_boolean(gst_structure_get_value(st, "final"));
        char *uri = select_link(cmd);
        if (uri != NULL && final) read_page(uri, 1);
    }

    return TRUE;
}

int speak_finalize(char *ssml) {
    while (read_keyboard) {
        if (!read_stdin(NULL, 0, ssml)) return 0;
    espeak_ng_STATUS result = espeak_ng_Synchronize();
    if (result != ENS_OK) {
        espeak_ng_PrintStatusCodeMessage(result, stderr, context);
        return 4;
    }

close:
    if (path_wav != NULL) sf_close(fd_wav);
    espeak_ng_Terminate();
    return 0;
}

/* Main driver */
int speak_err = 0;
struct session *session = NULL;
struct page *referer = NULL;

int use_espeak = 0;
char *logpath = NULL;
char *mimes;
FILE *fd_ssml = NULL;
FILE *fd_links = NULL;

void write_links() {
void write_links(FILE *fd_links, char **links) {
    for (int i = 0; strcmp(links[i], " ") != 0; i++) {
        fprintf(fd_links, "%s%c", links[i], (i % 3) == 2 ? '\n' : '\t');
    }
}

void read_page(char *uri, int read_input) {
    if (uri == NULL) return; // Leave things as-is...

    if (links != NULL) { // Ensure links is freed...
        for (int i = 0; strcmp(links[i], " ") != 0; i++) free(links[i]);
        free(links);
    }

    if (use_espeak && speak_err == 0) speak_text(uri, espeakRATE, 10);
    #ifdef WITH_SPEECHD
    else if (spd_conn != NULL) spd_say(spd_conn, SPD_MESSAGE, uri);
    #endif
    else printf("%s\n", uri);

    struct page *page = c_fetchURL(session, mimes, referer, uri);
    referer = page;
    char *ssml = c_renderDoc(session, page, use_espeak);
    links = c_extractLinks(page);

    if (logpath != NULL) c_writeLog(logpath, session);

    if (fd_ssml != NULL) fprintf(fd_ssml, "%s\n", ssml);
    if (read_keyboard && strcmp(uri, "about:welcome")) select_link("");
    if (fd_links != NULL) write_links(fd_links, links);
    if (use_espeak & speak_err == 0) speak(ssml, "main", NULL);
    #ifdef WITH_SPEECHD
    if (spd_conn != NULL) spd_say(spd_conn, SPD_MESSAGE, ssml);
    #endif

    c_freePage(page);
    //free(uri);

    if (read_input) {
        g_io_add_watch(g_io_channel_unix_new(0), G_IO_IN|G_IO_HUP, read_stdin, ssml);
    } else {
        free(ssml);
        for (int i = 0; strcmp(links[i], " ") != 0; i++) free(links[i]);
        free(links);
    }
}

FILE *parse_opt_file() {
    FILE *ret = optarg != NULL ? fopen(optarg, "w") : stdout;
    if (ret == NULL) {


@@ 415,24 323,30 @@ FILE *parse_opt_file() {
}

int main(int argc, char **argv) {
    int speak_err = 0;
    struct session *session = NULL;
    struct page *referer = NULL;

    int use_espeak = 0;
    char *logpath = NULL;
    char *mimes;
    FILE *fd_ssml = NULL;
    FILE *fd_links = NULL;

    hs_init(&argc, &argv);
    gst_init(&argc, &argv);
    GMainLoop *loop = g_main_loop_new(NULL, FALSE);
    guint bus_watch_id = 0;

    tcgetattr(0, &stored_settings);
    #ifdef WITH_SPEECHD
    SPDConnection *spd_conn = NULL;
    #endif
    GstElement *stt_pipeline = NULL;
    mimes = "text/html text/xml application/xml application/xhtml+xml text/plain";

    int c;
    opterr = 0;
    #ifdef WITH_SPEECHD
    while ((c = getopt(argc, argv, "xs::l::L:kKw::dvh")) != -1) {
    while ((c = getopt(argc, argv, "xs::l::L:kKw::dh")) != -1) {
    #else
    while ((c = getopt(argc, argv, "xs::l::kKw::vh")) != -1) {
    while ((c = getopt(argc, argv, "xs::l::kKw::h")) != -1) {
    #endif
        switch (c) {
        case 'x':


@@ 464,32 378,6 @@ int main(int argc, char **argv) {
            spd_set_data_mode(spd_conn, SPD_DATA_SSML);
            break;
        #endif
        case 'v':
            stt_pipeline = gst_pipeline_new("speech2text");
            GstElement *src = gst_element_factory_make("autoaudiosrc", "microphone");
            GstElement *convert = gst_element_factory_make("audioconvert", "convert");
            GstElement *resample = gst_element_factory_make("audioresample", "resample");
            GstElement *decoder = gst_element_factory_make("pocketsphinx", "asr");
            GstElement *sink = gst_element_factory_make("fakesink", "output");

            if (!src || !convert || !resample || !decoder || !sink) {
                fprintf(stderr, "Failed to initialize voice recognition. You'll just have to type your commands.\n");
                gst_object_unref(stt_pipeline);
                stt_pipeline = NULL;
                break;
            }

            // FIXME: Set something more appropriate
            g_object_set(G_OBJECT(decoder), "lmctl", "test.lmctl", NULL);
            g_object_set(G_OBJECT(decoder), "lmname", "tidigits", NULL);

            GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(stt_pipeline));
            bus_watch_id = gst_bus_add_watch(bus, stt_bus_call, NULL);
            gst_object_unref(bus);

            gst_bin_add_many(GST_BIN(stt_pipeline), src, convert, resample, decoder, sink, NULL);
            gst_element_link_many(src, convert, resample, decoder, sink, NULL);
            break;
        case '?':
            fprintf(stderr, "Invalid flag %c\n\n", optopt);
        case 'h':


@@ 543,28 431,54 @@ int main(int argc, char **argv) {
    referer = c_initialReferer();

    if (use_espeak) speak_err = speak_initialize();
    char *ssml, **links, *uri;
    int read_links = 0;
    if (optind >= argc) {
        // No URLs specified, restore previous session or goto about:welcome
        read_page(c_lastVisited("about:welcome"), 1);
    } else for (int i = optind; i < argc; i++) read_page(argv[i], i+1 == argc);
        uri = c_lastVisited("about:welcome");
        goto read_uri;
    }
    for (int i = optind; i < argc; i++) {
        uri = argv[i];

    g_main_loop_run(loop);
    if (bus_watch_id) g_source_remove(bus_watch_id);
    g_main_loop_unref(loop);
read_uri:
        if (use_espeak && speak_err == 0) speak_text(uri, espeakRATE, 10);
        #ifdef WITH_SPEECHD
        else if (spd_conn != NULL) spd_say(spd_conn, SPD_MESSAGE, uri);
        #endif
        else printf("%s\n", argv[i]);

        if (logpath != NULL) session = c_enableLogging(session);

        struct page *page = c_fetchURL(session, mimes, referer, uri);
        ssml = c_renderDoc(session, page, use_espeak);
        links = c_extractLinks(page);

        if (logpath != NULL) c_writeLog(logpath, session);

        if (fd_ssml != NULL) fprintf(fd_ssml, "%s\n", ssml);
        if (read_keyboard && use_espeak && strcmp(uri, "about:welcome") == 0) read_links = 1;
        if (fd_links != NULL) write_links(fd_links, links);
        if (use_espeak & speak_err == 0) speak(ssml, "main", NULL);
        #ifdef WITH_SPEECHD
        if (spd_conn != NULL) spd_say(spd_conn, SPD_MESSAGE, ssml);
        #endif

        c_freePage(page);
    }

    uri = NULL;
    if (read_links && speak_err == 0) {
        speak_err = espeak_ng_Synchronize();
        if (speak_err == 0) select_link(links, "");
    }
    if (use_espeak & speak_err == 0) speak_err = speak_finalize(ssml, links, &uri);
    if (uri != NULL) goto read_uri;

    #ifdef WITH_SPEECHD
    if (spd_conn != NULL) spd_close(spd_conn);
    #endif

    espeak_ng_STATUS result = espeak_ng_Synchronize();
    if (result != ENS_OK) {
        espeak_ng_PrintStatusCodeMessage(result, stderr, context);
        return 4;
    }
    if (path_wav != NULL) sf_close(fd_wav);
    espeak_ng_Terminate();

    if (stt_pipeline != NULL) gst_object_unref(GST_OBJECT(stt_pipeline));
    c_freePage(referer);
    c_freeSession(session);
    hs_exit();