每日 mark

SIGNAL=${SIGNAL:-TERM}

PIDS=$(jps -lm | grep -i 'kafka\.Kafka' | awk '{print $1}')
if [ -z "$PIDS" ]; then
echo "No kafka server to stop"
exit 1
else
kill -s $SIGNAL $PIDS
fi

PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}')

if [ -z "$PIDS" ]; then
echo "No kafka server to stop"
exit 1
else
kill -s TERM $PIDS
fi

1.面试题目集合

2.分布式实现算法

3.spark mllib

package sqlparser;
import java.io.*;
public class func{
    public static String readToString(String fileName) {
        String encoding = "UTF-8";
        File file = new File(fileName);
        Long filelength = file.length();
        byte[] filecontent = new byte[filelength.intValue()];
        try {
            FileInputStream in = new FileInputStream(file);
            in.read(filecontent);
            in.close();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        try {
            return new String(filecontent, encoding);
        } catch (UnsupportedEncodingException e) {
            System.err.println("The OS does not support " + encoding);
            e.printStackTrace();
            return null;
        }
    }}

秋天的颜色01 name+qq
Paic880807

import numpy as np
import pandas as pd
pd.set_option('display.max_columns', 10)
pd.set_option('expand_frame_repr', False)
def loadData():
    df_off = pd.read_csv(r'ccf_offline_stage1_train.csv')
    df_on  = pd.read_csv(r'ccf_online_stage1_train.csv')
    df_test  = pd.read_csv(r'ccf_offline_stage1_test_revised.csv')

    return df_off ,df_on ,df_test
    # return df_off[:10],df_on[:10],df_test[:10]
#    User_id  Merchant_id  Coupon_id Discount_rate  Distance  Date_received        Date
# 0  1439408         2632        NaN           NaN       0.0            NaN  20160217.0
# 1  1439408         4663    11002.0        150:20       1.0     20160528.0         NaN
# 2  1439408         2632     8591.0          20:1       0.0     20160217.0         NaN
# 3  1439408         2632     1078.0          20:1       0.0     20160319.0         NaN
# 4  1439408         2632     8591.0          20:1       0.0     20160613.0         NaN
#     User_id  Merchant_id  Action  Coupon_id Discount_rate  Date_received        Date
# 0  13740231        18907       2  100017492        500:50     20160513.0         NaN
# 1  13740231        34805       1        NaN           NaN            NaN  20160321.0
# 2  14336199        18907       0        NaN           NaN            NaN  20160618.0
# 3  14336199        18907       0        NaN           NaN            NaN  20160618.0
# 4  14336199        18907       0        NaN           NaN            NaN  20160618.0
#    User_id  Merchant_id  Coupon_id Discount_rate  Distance  Date_received
# 0  4129537          450       9983          30:5       1.0       20160712
# 1  6949378         1300       3429          30:5       NaN       20160706
# 2  2166529         7113       6928        200:20       5.0       20160727
# 3  2166529         7113       1808        100:10       5.0       20160727
# 4  6172162         7605       6500          30:1       2.0       20160708
#  0    977900 'User_id','Merchant_id','Coupon_id','Discount_rate','Date_received','Date'
# -1    701602
#  1     75382
# Name: label, dtype: int64
# -1    10557469
#  0      655898
#  1      216459
df_off,df_on,df_test = loadData()

df_off['label'] = -1
df_off.loc[df_off['Coupon_id'].notnull() & df_off['Date'].notnull(),'label'] = 1
df_off.loc[df_off['Coupon_id'].notnull() & df_off['Date'].isnull(),'label'] = 0

df_on['label'] = -1
df_on.loc[df_on['Coupon_id'].notnull() & df_on['Date'].notnull(),'label'] = 1
df_on.loc[df_on['Coupon_id'].notnull() & df_on['Date'].isnull(),'label'] = 0

real_off = df_off[df_off.label.isin([0,1])]
real_on = df_on[df_on.label.isin([0,1])]

real_all = pd.concat([real_off[['User_id','Merchant_id','Coupon_id','Discount_rate','Date_received','Date','label']],real_on[['User_id','Merchant_id','Coupon_id','Discount_rate','Date_received','Date','label']]])
print (real_all.iloc[:,0].size,real_off.iloc[:,0].size,real_on.iloc[:,0].size)
real_all['tmp'] = pd.to_datetime(  (real_all['Date_received'].astype(int).apply(str)))
real_all['weekday'] = real_all['tmp'].dt.weekday_name
print (real_all.groupby(['weekday','label']).count())
# pd.pivot_table(real_all,values = 'label',index='weekday')

http://mooc.study.163.com/university/deeplearning_ai#/c

__author__ = 'Administrator'
import time
import pandas as  pd
def runtime(func):
    def wrapper(*args,**kwargs):
        t1 = time.time()
        func(*args,**kwargs)
        t2=time.time()
        print ("{0}函数调用耗时：{1:.2f}".format (func.__name__,t2-t1))
    return wrapper()

def loadData():
    df_train = pd.read_csv("weibo_train_data.txt",header = None,sep = '\t')
    df_train.columns = ["uid","mid","date","forward","comment","like","content"]
    df_test = pd.read_csv("weibo_predict_data.txt",header = None,sep = '\t')
    df_test.columns = ["uid","mid","date","content"]
    return df_train,df_test

def dataProcess(data):
    df = data.groupby('uid').agg(['median','mean'])
    df.columns =[ 'forward_median','forward_mean','comment_median','comment_mean','like_median','like_mean']
    train_stat = df.apply(pd.Series.round)
    uid_dict = {}
    for uid,row in df.iterrows():
        uid_dict[uid] = row
    return  uid_dict

def fill_with_fixed_data(f,c,l):
    df_train,df_test = loadData()
    df1 = df_test[['uid','mid']]
    df1['forward']  = f
    df1['comment']  = c
    df1['like']  =  l
    result = []
    for _,row in df1.iterrows():
        result.append("{0}\t{1}\t{2},{3},{4}\n".format(row[0],row[1],row[2],row[3],row[4]))
    filename  = "weibo_predict_{}_{}_{}.txt".format( f,c,l)
    f= open(filename,'w')
    f.writelines(result)
    f.close()
    return result

def fill_with_stat_data(stat = 'median'):
    df_train,df_test = loadData()
    uid_dict = dataProcess(df_train)
    df1 = df_test[['uid','mid']]
    forward,comment,like = [],[],[]
    print (uid_dict)
    for uid in df_test['uid']:
        if uid in uid_dict:
            forward.append(int(uid_dict[uid]["forward_"+stat]))
            comment.append(int(uid_dict[uid]["comment_"+stat]))
            like.append(int(uid_dict[uid]["like_"+stat]))
        else:
            forward.append(0)
            comment.append(0)
            like.append(0)
    df1['forward']  = forward
    df1['comment']  = comment
    df1['like']  =  like
    result = []
    for _,row in df1.iterrows():
        result.append("{0}\t{1}\t{2},{3},{4}\n".format(row[0],row[1],row[2],row[3],row[4]))
    filename  = "weibo_predict_{}.txt".format( stat)
    f= open(filename,'w')
    f.writelines(result)
    f.close()
    return result

fill_with_stat_data(  )

from numpy import *
import operator
from functools import reduce
def loadDataSet():
    postingList=[['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                 ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                 ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                 ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                 ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                 ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
    classVec = [0,1,0,1,0,1]    #1 is abusive, 0 not
    return postingList,classVec

def createVocabList(dataSet):
    vocabSet = set(reduce(operator.add, dataSet))
    return list(vocabSet)

def setOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1
        else: print ("the word: %s is not in my Vocabulary!" % word)
    return returnVec

from numpy import *
from os import listdir
from numpy.ma import zeros
class kNN(object):
    def __init__(self,**kwargs):
        pass

    def data2matrix(self):
        fr = open('G:\zqh_work\ML\datasets\ml_ac\Ch02\datingTestSet2.txt')
        lines = fr.readlines()
        line_num = len(lines)
        mat = zeros((line_num,3))
        labels = []
        for i in range(line_num):
            mat[i] = lines[i].strip().split('\t')[0:3]
            labels[i] = lines[i].strip().split('\t')[-1]
        return mat,labels

    def norm_data(self,mat):
        max = mat.max(0)
        min = mat.min(0)
        diff = max - min
        rows = mat.shape[0]
        norm_mat = (mat - tile(min,(rows,1)))/tile(diff,(rows,1))
        return norm_mat

    def classify(self,inX,norm_mat,labels,k):
        rows = norm_mat.shape[0]
        for i in range(rows):

    def test(self):
        mat,labels  = self.data2matrix()
        norm_mat = self.norm_data(mat)
        print (norm_mat)
        return norm_mat

if    __name__ == '__main__':
    knn=kNN()
    knn.test()

http://keras-cn.readthedocs.io/en/latest/

http://wiki.jikexueyuan.com/project/tensorflow-zh/get_started/introduction.html

https://segmentfault.com/a/1190000002766035

https://www.jianshu.com/p/8bb456cb7c77

http://cache.baiducontent.com/c?m=9f65cb4a8c8507ed4fece763104d96275e03c1743ca083572c85c91f84642c1c0733fee37c6243198385212240f8543d8883560b200356b799c28f4ac9fecf6879877a74250b873105d36eb8ca36768373c100beb81897adf04584afa2929d07139344040a97f0fc4d01648b2cae033093b1993f025e60eda76734b81f2c74c33441c650f997256f77d1b189081b837d867610e7ef68f52913c548e2485b7702fd0ca6092131309758268f1e6e4585ea2dbb7d3306&p=c2769a479d9e0bb312bd9b7e0d1488&newp=8465c64ad49506e42abd9b7e0d1496231610db2151d7d4146b82c825d7331b001c3bbfb423251003d2c0776600af495ee8f5367630032ba3dda5c91d9fb4c57479de607f02&user=baidu&fm=sc&query=org%2Eapache%2Espark%2Esql%2Eexecution%2EBufferedRowIterator%2EhasNext&qid=853831ee00006451&p1=7

org.apache.spark.sql.execution.BufferedRowIterator.hasNext

spark.write是否是分布式写？

scala 事务控制？

yarn的web ui 配置，rm是哪台机器 ？

为什么不用yarn-cluster？不好收集日志？

executor 日志 如何 查看 ？

spark的几个配置文件适用情形？

https://www.cnblogs.com/sorco/p/7070922.html

http://hongjiang.info/scala/    写点什么

spark executor 日志：$SPARK_HOME/work/$app_id/$executor_id/stdout

总结一下Spark中各个角色的JVM参数设置：

(1)Driver的JVM参数：
-Xmx，-Xms，如果是yarn-client模式，则默认读取spark-env文件中的SPARK_DRIVER_MEMORY值，-Xmx，-Xms值一样大小；如果是yarn-cluster模式，则读取的是spark-default.conf文件中的spark.driver.extraJavaOptions对应的JVM参数值。
PermSize，如果是yarn-client模式，则是默认读取spark-class文件中的JAVA_OPTS="-XX:MaxPermSize=256m $OUR_JAVA_OPTS"值；如果是yarn-cluster模式，读取的是spark-default.conf文件中的spark.driver.extraJavaOptions对应的JVM参数值。
GC方式，如果是yarn-client模式，默认读取的是spark-class文件中的JAVA_OPTS；如果是yarn-cluster模式，则读取的是spark-default.conf文件中的spark.driver.extraJavaOptions对应的参数值。
以上值最后均可被spark-submit工具中的--driver-java-options参数覆盖。

(2)Executor的JVM参数：
-Xmx，-Xms，如果是yarn-client模式，则默认读取spark-env文件中的SPARK_EXECUTOR_MEMORY值，-Xmx，-Xms值一样大小；如果是yarn-cluster模式，则读取的是spark-default.conf文件中的spark.executor.extraJavaOptions对应的JVM参数值。
PermSize，两种模式都是读取的是spark-default.conf文件中的spark.executor.extraJavaOptions对应的JVM参数值。
GC方式，两种模式都是读取的是spark-default.conf文件中的spark.executor.extraJavaOptions对应的JVM参数值。

(3)Executor数目及所占CPU个数
如果是yarn-client模式，Executor数目由spark-env中的SPARK_EXECUTOR_INSTANCES指定，每个实例的数目由SPARK_EXECUTOR_CORES指定；如果是yarn-cluster模式，Executor的数目由spark-submit工具的--num-executors参数指定，默认是2个实例，而每个Executor使用的CPU数目由--executor-cores指定，默认为1核。
每个Executor运行时的信息可以通过yarn logs命令查看到，类似于如下：

14/08/13 18:12:59 INFO org.apache.spark.Logging$class.logInfo(Logging.scala:58): Setting up executor with commands: List($JAVA_HOME/bin/java, -server, -XX:OnOutOfMemoryError='kill %p', -Xms1024m -Xmx1024m , -XX:PermSize=256M -XX:MaxPermSize=256M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -Xloggc:/tmp/spark_gc.log, -Djava.io.tmpdir=$PWD/tmp, -Dlog4j.configuration=log4j-spark-container.properties, org.apache.spark.executor.CoarseGrainedExecutorBackend, akka.tcp://spark@sparktest1:41606/user/CoarseGrainedScheduler, 1, sparktest2, 3, 1>, <LOG_DIR>/stdout, 2>, <LOG_DIR>/stderr)

其中，akka.tcp://spark@sparktest1:41606/user/CoarseGrainedScheduler表示当前的Executor进程所在节点，后面的1表示Executor编号，sparktest2表示ApplicationMaster的host，接着的3表示当前Executor所占用的CPU数目。

先在spark-env.sh 增加SPARK_HISTORY_OPTS；

然后启动start-history-server.sh服务；

就可以看到启动了HistoryServer进程，且监听端口是18080。

之后就可以在web上使用http://hostname:18080愉快的玩耍了。

作者：俺是亮哥
链接：https://www.jianshu.com/p/65a3476757a5
來源：简书
著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。

问题的症结就在于：闭包没有办法序列化。在这个例子里，闭包的范围是：函数parser以及它所依赖的一个隐式参数: formats , 而问题就出在这个隐式参数上， 它的类型是DefaultFormats，这个类没有提供序列化和反序列自身的说明，所以Spark无法序列化formats，进而无法将task推送到远端执行。

隐式参数formats是为extract准备的，它的参数列表如下：

org.json4s.ExtractableJsonAstNode#extract[A](implicit formats: Formats, mf: scala.reflect.Manifest[A]): A = ...

• 1

找到问题的根源之后就好解决了。实际上我们根本不需要序列化formats， 对我们来说，它是无状态的。所以，我们只需要把它声明为一个全局静态的变量就可以绕过序列化。所以改动的方法就是简单地把implicit val formats = DefaultFormats的声明从方法内部迁移到App Object的字段位置上即可。

{
"color_scheme": "Packages/Color Scheme - Default/Monokai.tmTheme",
"font_size": 13,
"ignored_packages":
[
"Vintage"
],
"preview_on_click": false,
"word_wrap": "true"
}

ambari  部署

scala : 静态方法，单例对象，伴生对象

spark job server

spark etl

spark 资源限制

yarn   资源队列限制  用户限制

算法

hdfs 挂载  像访问自己的目录

f5

keepalived

sdg agent 采集redolog

active mq vs kafka

tomcat 备份

finixs

组的概念？ hadoop组

hue权限控制

spark thirft service

spark submit

ranger 只控制sdo？不能控制命令行？

sms抓取元数据与ctm 配对

ranger ............

scala 闭包

java 内部类

/etc/security/limits.conf

同步命令：scp –r /seabox/develop/  26.6.0.141:/seabox

谓词下推

整理要了解的业务知识

(select z.*,row_number() over(partition by z.deal_no order by z.biz_date desc) rn
from bridge.summit_i_repo_general_info_ib z
where z.deal_no not in (select distinct deal_no from bridge.summit_i_repo_general_info_ib where deal_status in ('3', '4') and biz_date <= '{DATE_YYYYMMDD}')
)

如何匹配 ：received 数字 rows  ？

START_TIME=`date "+%Y-%m-%d %H:%M:%S"`   ????????

awk -F: '{print"用户帐号："$1}'

sqoop 各参数

kettle 导出为xml文件

http://confluence.paic.com.cn:6060/pages/viewpage.action?pageId=2132765

http://www.docin.com/p-1354952858.html

oracle 连接：

JDBC

ODBC

OCI

JNDI

http://logging.apache.org/log4j/2.x/

查看当前进程：ps

可以用来查找某一应用运行在哪里 ：ps -aux | grep hive

flume 收集log4j日志的例子：

http://blog.csdn.net/nsrainbow/article/details/36875123

H75244

Uy1caTod6Hgb

建表时没有定义分隔符，分桶等，在表建成之后还能不能再加上？

val dfa = sc.parallelize(List(("1", "aa", "ab"), ("2", "bb", "bb"),("4", "dd", "dd"))).toDF("key", "val1", "val2")
val dfb = sc.parallelize(List(("1", "aa", "ab"), ("2", "bb", "cc"), ("3", "cc", "cc"))).toDF("key", "val1", "val2")
val dfc = sc.parallelize(List( ("key"),("val1"))).toDF("pkey")
val rv1 = dfb.join(dfa, dfa("key") === dfb("key") and dfa("val1") === dfb("val1"), "outer").show()
val tmp = dfc.select("pkey").collect().map(_(0).toString())
val mid = new Array[org.apache.spark.sql.Column](tmp.length)
for (i<- 0 until tmp.length) mid(i)=dfa(tmp(i))===dfb(tmp(i))
val rv2 = dfb.join(dfa, mid.reduce(_ and _), "outer")

val cols = dfb.columns
val all_col = new Array[org.apache.spark.sql.Column](cols.length)
for (i <- 0 until cols.length) all_col(i)=when(dfb("key").isNull, dfa(cols(i))).otherwise(dfb(cols(i))).as(cols(i))

val rv3 = rv2.select(all_col:_*).show()

rv2.select(when(dfb("key").isNull, dfa("key")).otherwise(dfb("key")).as("key"))

import scala.collection.mutable.ArrayBuffer
val cols = dfb.columns
val a=dfb.dtypes
val b = new ArrayBuffer[String]()
for (i <- a if i._2=="IntegerType") b+=i._1

val numArray = b.toArray
val num_col = new Array[org.apache.spark.sql.Column](numArray.length)
for (i <- 0 until numArray.length) num_col(i)=when(dfb("key").isNull, lit(0)).otherwise(dfb(numArray(i))).as(numArray(i))

val strArray = cols.filterNot(numArray.contains(_))
val str_col = new Array[org.apache.spark.sql.Column](strArray.length)
for (i <- 0 until strArray.length) str_col(i)=when(dfb("key").isNull, dfa(strArray(i))).otherwise(dfb(strArray(i))).as(strArray(i))

val rv3 = rv2.select((num_col ++ str_col):_*)
rv3.show()