Slony-I的 RemoteWorker重试调查

客户的问题是：

向Slony-I运行环境中，增加新的slaveDB节点的时候发生错误。

log中反复出现错误，然后再重新开始（重新开始部分的log省略）：

CONFIG remoteWorkerThread_1: connected to provider DB
CONFIG remoteWorkerThread_1: prepare to copy table "tst"."a_tbl"
CONFIG remoteWorkerThread_1: prepare to copy table "tst"."b_tbl"
CONFIG remoteWorkerThread_1: prepare to copy table "tst"."c_tbl"
CONFIG remoteWorkerThread_1: all tables for set  found on subscriber
CONFIG remoteWorkerThread_1: copy sequence "tst"."a_no_seq"
CONFIG remoteWorkerThread_1: copy sequence "tst"."b_no_seq"
CONFIG remoteWorkerThread_1: copy sequence "tst"."c_no_seq"

CONFIG remoteWorkerThread_1: copy table "tst"."a_tbl"
CONFIG remoteWorkerThread_1: Begin COPY of table "tst"."a_tbl"
NOTICE:  truncate of "tst"."a_tbl" succeeded
CONFIG remoteWorkerThread_1:  bytes copied for table "tst"."a_tbl"
CONFIG remoteWorkerThread_1: 27.97 seconds to copy table "tst"."a_tbl"

CONFIG remoteWorkerThread_1: copy table "tst"."b_tbl"
CONFIG remoteWorkerThread_1: Begin COPY of table "tst"."b_tbl"
ERROR  remoteWorkerThread_1: "select "_mycluster".copyFields(2);" 

WARN   remoteWorkerThread_1: data copy for set  failed  times - sleep  seconds
NOTICE:  Slony-I: Logswitch to sl_log_2 initiated
CONTEXT:  SQL statement "SELECT "_mycluster".logswitch_start()"

经过查阅资料，并且和客户沟通，发现是他们的网络环境有问题：原有节点所在网段和新增节点不在一个网段。而他们又使用了网络工具来监控网络，在某些特定情况下，网络工具会切点网络连接。

正式此原因，导致出错。然后我进行了代码分析，发现remoteworker是很勤劳的，如果发生了通讯错误，它会反复重试的：

remoteWorkerThread_main函数的while循环，就会完成这个工作。

/* ----------
 * slon_remoteWorkerThread
 *
 * Listen for events on the local database connection. This means, events
 * generated by the local node only.
 * ----------
 */
void *
remoteWorkerThread_main(void *cdata)
{
    …
    /*
     * Work until shutdown or node destruction
     */
    while (true)
      {
        …
        /*
         * Event type specific processing
         */
        if (strcmp(event->ev_type, "SYNC") == )
        {
            …
        }
        else    /* not SYNC */
        {
            …
            /*
             * Simple configuration events. Call the corresponding runtime
             * config function, add the query to call the configuration event
             * specific stored procedure.
             */
            if (strcmp(event->ev_type, "STORE_NODE") == )
            {
                …
            }
            …
            else if (strcmp(event->ev_type, "ENABLE_SUBSCRIPTION") == )
            {
                …
                int         copy_set_retries = ;
                …                                    

                if (sub_receiver == rtcfg_nodeid &&
                    event->ev_origin == node->no_id)
                {
                    ScheduleStatus            sched_rc;
                    int            sleeptime = ;
                    …
                    while (true) 
                    {
                        …
                        /*
                         * If the copy succeeds, exit the loop and let the
                         * transaction commit.
                         */ 
                        if (copy_set(node, local_conn, sub_set, event) == )
                        {
                            …
                            copy_set_retries = ;
                            break;
                        }
                        copy_set_retries++;                            

                        /*
                         * Data copy for new enabled set has failed. Rollback
                         * the transaction, sleep and try again.
                         */
                        slon_log(SLON_WARN, "remoteWorkerThread_%d: "
                                 "data copy for set %d failed %d times - "
                                 "sleep %d seconds\n",
                                 node->no_id, sub_set, copy_set_retries,
                                 sleeptime);
                        …      
                    }
                }
                else
                {
                    …
                }
                …
            }
            …
            else
            {
                …
            }                                        

            /*
             * All simple configuration events fall through here. Commit the
             * transaction.
             */
            …
        }
        …
    }
    …
}                                                    

/* ----------
 * copy_set
 * ----------
 */
static int
copy_set(SlonNode *node, SlonConn *local_conn, int set_id,
         SlonWorkMsg_event *event)
{
    …
    /*
     * Connect to the provider DB
     */
    …
    slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
             "connected to provider DB\n",
             node->no_id);
    …
    /*
     * For each table in the set
     */
    for (tupno1 = ; tupno1 < ntuples1; tupno1++)
    {
        char       *tab_fqname = PQgetvalue(res1, tupno1, );                                        

        gettimeofday(&tv_start2, NULL);
        slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
                 "prepare to copy table %s\n",
                 node->no_id, tab_fqname);                                    

        (void) slon_mkquery(&query3, "select * from %s limit 0;",
                            tab_fqname);
        res2 = PQexec(loc_dbconn, dstring_data(&query3));
        …
    }
    …
    slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
             "all tables for set %d found on subscriber\n",
             node->no_id, set_id);
    …
    for (tupno1 = ; tupno1 < ntuples1; tupno1++)
    {
        …
        slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
                 "copy sequence %s\n",
                 node->no_id, seq_fqname);
        …
    }
    …                                                

    /*
     * For each table in the set
     */
    for (tupno1 = ; tupno1 < ntuples1; tupno1++)
    {
        …
        slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
                 "copy table %s\n",
                 node->no_id, tab_fqname);
        …
        if (omit_copy) {
            …
        } else {
            slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
                 "Begin COPY of table %s\n",
                 node->no_id, tab_fqname);                                    

            (void) slon_mkquery(&query2, "select %s.copyFields(%d);",
                            rtcfg_namespace, tab_id);                        

            res3 = PQexec(pro_dbconn, dstring_data(&query2));                                        

            if (PQresultStatus(res3) != PGRES_TUPLES_OK)
            {
                slon_log(SLON_ERROR, "remoteWorkerThread_%d: \"%s\" %s\n",
                         node->no_id, dstring_data(&query2),
                         PQresultErrorMessage(res3));
                …
                return -;
            }                                        

        …
        slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
                 INT64_FORMAT " bytes copied for table %s\n",
                 node->no_id, copysize, tab_fqname);
        …
        slon_log(SLON_CONFIG, "remoteWorkerThread_%d: "
                 "%.3f seconds to copy table %s\n",
                 node->no_id,
                 TIMEVAL_DIFF(&tv_start2, &tv_now), tab_fqname);
    }
    …
    return ;
}