使用 Python 与 Adapter EIM 实时绘制音乐 Spectrogram

使用 python 与 madmom 包分析音频数据,通过 adapter eim 插件将实时数据发送给 Scratch,将 Scratch 作为 UI 实时画图呈现。

Scratch 代码在此

import madmom
import numpy as np
import time
from codelab_adapter_client import AdapterNode

class MyNode(AdapterNode):
    NODE_ID = "eim/list_data"

    def __init__(self):
        super().__init__()
    
    def send_data(self, content):
        message = self.message_template()
        message["payload"]["content"] = content
        self.publish(message)

node = MyNode()
node.receive_loop_as_thread()
time.sleep(0.1)


kwargs = dict(
    sample_rate=44100,
    num_channels=1,
    frame_size=2048,
    hop_size=441,
    filterbank=madmom.audio.filters.MelFilterbank,
    num_bands=12
    
)
stream=signal.Stream(**kwargs)


spec_list = [i for i in range(10)]

n = 0

try:
    for frames in stream:  
        fs = madmom.audio.signal.FramedSignal(frames,**kwargs)
        stft = madmom.audio.stft.STFT(fs)
        spec = madmom.audio.spectrogram.LogarithmicFilteredSpectrogram(stft, **kwargs)
        i = n%2
        if i == 0:
            for j in range(len(spec)):
                spec_list[i*5+j]=np.ndarray.tolist(spec[j])
            n = n + 1
        elif i == 1:
            for j in range(len(spec)):
                spec_list[i*5+j]=np.ndarray.tolist(spec[j])
            #print(lindaout)
            node.send_data(spec_list)
            n = n + 1   
            
            

except KeyboardInterrupt:
    print('interrupt by user')

1赞

对应 Scratch 项目在此

以下是自动启动对应 Scratch 项目版的代码:

import madmom
import numpy as np
import webbrowser
import time
from codelab_adapter_client import AdapterNode


# EIM 初始化
class MyNode(AdapterNode):
    NODE_ID = "eim/spectrogram"

    def __init__(self):
        super().__init__()
        self.is_ready = False
    
    def send_data(self, content):
        message = self.message_template()
        message["payload"]["content"] = content
        self.publish(message)
        
    def extension_message_handle(self, topic, payload):
        self.logger.info(f'the message payload from scratch: {payload}')
        content = payload["content"]
        if content == "ready":
            self.is_ready = True  

node = MyNode()
node.receive_loop_as_thread()
time.sleep(0.1)



# 在新的浏览器窗口自动打开对应 Scratch 项目,使用 EIM 发送消息使 Scratch 项目待命,同时等待项目加载成功后的返回消息
webbrowser.open('https://create.codelab.club/projects/9942/editor/', new=1)

print("Waiting for Scratch.")

while not node.is_ready:  
    node.send_data("go")
    time.sleep(0.5)

print("scratch is ready")     
    


# 使用 Madmom 实时抓取音频流数据,做 sftf 并生成 sepctrogram
kwargs = dict(
    sample_rate=44100,                        # 采样率
    num_channels=1,                           # 单声道
    frame_size=2048,                          # 以 2048 sample 为一个 frame 做 stft 分析,同时也是窗口大小
    hop_size=441,                             # 间隔 441 sample 有重合的取 frame 做 stft 分析
    filterbank=madmom.audio.filters.MelFilterbank,          # 对频率做 MelFilter
    num_bands=12                              # 分为 12 个频段
    
)


stream=madmom.audio.signal.Stream(**kwargs)

spec_list = [i for i in range(10)]            # 以 10 个时间点的结果为单位向 Scratch 发送列表数据

n = 0

try:
    for frames in stream:  
        fs = madmom.audio.signal.FramedSignal(frames,**kwargs)   # 将以 sample 为单位的信号转为以 frame 为单位的信号
        stft = madmom.audio.stft.STFT(fs)                        # 做 stft 分析
        spec = madmom.audio.spectrogram.LogarithmicFilteredSpectrogram(stft, **kwargs)     # 对频率做 MelFilter 同时对强度值做对数转换
        i = n%2
        if i == 0:                                               # 累积两个 frame 的结果发送给 Scratch
            for j in range(len(spec)):
                spec_list[i*5+j]=np.ndarray.tolist(spec[j])
            n = n + 1
        elif i == 1:
            for j in range(len(spec)):
                spec_list[i*5+j]=np.ndarray.tolist(spec[j])
            print(spec_list)
            node.send_data(spec_list)
            n = n + 1   
            
            

except KeyboardInterrupt:
    stream.close()
    print('interrupt by user')

1赞

使用 thermal colormap 优化频谱的色彩呈现,对应 Scratch 项目在此

# 暂时先把列表导入放在这里,按说应该是作为单独的文件与代码分离开,在代码中导入引用,但这部分还不清楚怎么操作,后面再改。
# 这一栏需要运行,后面代码要用到这个列表 
# 这个 map 由上图所示 256 个颜色构成,每个颜色对应的 RGB 值存在下面列表中
# 这个 map 具有知觉一致性(perceptually uniform),



thermal_rgb = [[ 0.01555601, 0.13824425, 0.20181089],
           [ 0.01620184, 0.14105074, 0.20897651],
           [ 0.01685649, 0.14382701, 0.21623868],
           [ 0.0175264 , 0.14657173, 0.2235997 ],
           [ 0.01821872, 0.14928346, 0.23106187],
           [ 0.01894138, 0.15196073, 0.23862748],
           [ 0.01969968, 0.15460145, 0.24630497],
           [ 0.02050332, 0.15720378, 0.25409711],
           [ 0.02136721, 0.15976645, 0.26199915],
           [ 0.02230341, 0.16228755, 0.27001321],
           [ 0.0233252 , 0.16476505, 0.27814139],
           [ 0.02444728, 0.16719678, 0.28638573],
           [ 0.02568582, 0.16958042, 0.29474817],
           [ 0.02705867, 0.1719135 , 0.30323056],
           [ 0.02858553, 0.17419338, 0.31183463],
           [ 0.03028808, 0.17641726, 0.32056191],
           [ 0.03219022, 0.17858215, 0.32941369],
           [ 0.03431826, 0.18068487, 0.33839101],
           [ 0.03670118, 0.18272205, 0.34749451],
           [ 0.03937082, 0.18469014, 0.35672441],
           [ 0.04230474, 0.18658537, 0.36608039],
           [ 0.04544128, 0.1884038 , 0.37556146],
           [ 0.04879889, 0.1901413 , 0.38516584],
           [ 0.05238565, 0.19179358, 0.39489082],
           [ 0.05620897, 0.19335621, 0.40473254],
           [ 0.06027561, 0.19482469, 0.41468582],
           [ 0.06459519, 0.19618775, 0.42477146],
           [ 0.06917294, 0.19744583, 0.43495728],
           [ 0.07401398, 0.19859437, 0.44523225],
           [ 0.07912633, 0.19962514, 0.45559656],
           [ 0.08452075, 0.20052842, 0.46605087],
           [ 0.09019392, 0.20130794, 0.47654788],
           [ 0.09616431, 0.20194725, 0.48710445],
           [ 0.1024254 , 0.20245202, 0.49766462],
           [ 0.10899443, 0.20280889, 0.50822709],
           [ 0.11585974, 0.20302735, 0.51872453],
           [ 0.12304243, 0.2030938 , 0.52914838],
           [ 0.13052767, 0.20302178, 0.53942012],
           [ 0.13830991, 0.20281956, 0.54947678],
           [ 0.14637971, 0.20250018, 0.55924613],
           [ 0.15471863, 0.20208507, 0.56864226],
           [ 0.16329705, 0.20160546, 0.57756769],
           [ 0.17207282, 0.20110273, 0.58591838],
           [ 0.18099176, 0.20062705, 0.59359182],
           [ 0.18999022, 0.20023416, 0.60049707],
           [ 0.19899974, 0.19998029, 0.60656515],
           [ 0.20795298, 0.19991643, 0.61175706],
           [ 0.21678952, 0.20008303, 0.61606739],
           [ 0.22546043, 0.20050671, 0.61952279],
           [ 0.23393063, 0.2011992 , 0.62217608],
           [ 0.24217907, 0.20215872, 0.62409793],
           [ 0.25019713, 0.20337273, 0.62536824],
           [ 0.25798611, 0.20482134, 0.62606879],
           [ 0.26555442, 0.20648047, 0.62627797],
           [ 0.27291504, 0.20832446, 0.62606758],
           [ 0.28008339, 0.21032792, 0.62550133],
           [ 0.28706751, 0.21246914, 0.624642  ],
           [ 0.29388514, 0.21472543, 0.62353802],
           [ 0.30055772, 0.21707565, 0.62222519],
           [ 0.30708438, 0.21950598, 0.6207555 ],
           [ 0.31349163, 0.22199831, 0.61914522],
           [ 0.31977984, 0.22454204, 0.61743434],
           [ 0.32596955, 0.22712412, 0.61563293],
           [ 0.33205791, 0.22973736, 0.61377715],
           [ 0.33806601, 0.23237153, 0.61186606],
           [ 0.34399174, 0.23502151, 0.60992807],
           [ 0.34984607, 0.23768089, 0.60796949],
           [ 0.35563846, 0.2403444 , 0.60599535],
           [ 0.36136863, 0.24300892, 0.60402414],
           [ 0.36704345, 0.24567078, 0.60206076],
           [ 0.37267088, 0.2483267 , 0.60010594],
           [ 0.3782548 , 0.25097436, 0.59816563],
           [ 0.38379608, 0.25361228, 0.59624988],
           [ 0.38929883, 0.25623877, 0.5943618 ],
           [ 0.3947691 , 0.25885207, 0.59249964],
           [ 0.40020989, 0.26145107, 0.59066609],
           [ 0.40562409, 0.26403484, 0.58886331],
           [ 0.41101442, 0.26660262, 0.58709291],
           [ 0.41638322, 0.26915383, 0.5853566 ],
           [ 0.42173132, 0.27168825, 0.58365836],
           [ 0.42706343, 0.27420503, 0.5819941 ],
           [ 0.43238181, 0.2767038 , 0.58036393],
           [ 0.43768862, 0.27918426, 0.57876764],
           [ 0.44298597, 0.28164615, 0.5772048 ],
           [ 0.44827587, 0.28408925, 0.57567472],
           [ 0.45356026, 0.28651338, 0.57417648],
           [ 0.45884104, 0.28891838, 0.57270899],
           [ 0.46412001, 0.2913041 , 0.57127095],
           [ 0.46939893, 0.29367044, 0.5698609 ],
           [ 0.47467951, 0.29601729, 0.56847725],
           [ 0.47996336, 0.29834456, 0.56711823],
           [ 0.48525207, 0.30065217, 0.56578195],
           [ 0.49054716, 0.30294008, 0.5644664 ],
           [ 0.49585007, 0.30520822, 0.56316946],
           [ 0.50116221, 0.30745656, 0.56188888],
           [ 0.50648491, 0.30968508, 0.56062233],
           [ 0.51181943, 0.31189376, 0.55936737],
           [ 0.51716699, 0.31408263, 0.55812148],
           [ 0.52252871, 0.31625172, 0.55688205],
           [ 0.52790567, 0.31840106, 0.55564642],
           [ 0.53329886, 0.32053073, 0.55441182],
           [ 0.5387092 , 0.32264084, 0.55317546],
           [ 0.54413753, 0.3247315 , 0.55193448],
           [ 0.54958461, 0.32680287, 0.55068596],
           [ 0.55505113, 0.32885514, 0.54942696],
           [ 0.56053768, 0.33088852, 0.54815451],
           [ 0.56604476, 0.33290327, 0.54686563],
           [ 0.57157236, 0.33489987, 0.54555812],
           [ 0.57712131, 0.33687845, 0.54422801],
           [ 0.58269186, 0.33883938, 0.54287227],
           [ 0.58828416, 0.34078308, 0.54148789],
           [ 0.59389827, 0.34271002, 0.54007187],
           [ 0.59953415, 0.34462069, 0.53862124],
           [ 0.60519167, 0.34651565, 0.53713308],
           [ 0.61087061, 0.34839552, 0.5356045 ],
           [ 0.61657065, 0.35026093, 0.53403268],
           [ 0.62229138, 0.3521126 , 0.53241484],
           [ 0.62803229, 0.35395127, 0.53074831],
           [ 0.63379279, 0.35577776, 0.52903045],
           [ 0.63957219, 0.35759292, 0.52725878],
           [ 0.64536975, 0.35939765, 0.52543076],
           [ 0.6511847 , 0.36119285, 0.52354391],
           [ 0.6570161 , 0.36297956, 0.52159597],
           [ 0.66286293, 0.36475881, 0.51958481],
           [ 0.6687241 , 0.36653173, 0.51750839],
           [ 0.67459845, 0.36829947, 0.5153648 ],
           [ 0.68048472, 0.37006325, 0.51315222],
           [ 0.68638162, 0.37182432, 0.51086898],
           [ 0.69228775, 0.37358401, 0.50851352],
           [ 0.69820166, 0.37534367, 0.50608439],
           [ 0.70412186, 0.37710473, 0.50358028],
           [ 0.71004688, 0.37886859, 0.50099965],
           [ 0.71597525, 0.3806367 , 0.49834106],
           [ 0.72190503, 0.38241075, 0.4956041 ],
           [ 0.72783443, 0.38419239, 0.49278795],
           [ 0.73376164, 0.3859833 , 0.4898919 ],
           [ 0.73968475, 0.38778523, 0.48691538],
           [ 0.7456018 , 0.38960001, 0.48385794],
           [ 0.75151077, 0.39142951, 0.48071927],
           [ 0.75740957, 0.39327566, 0.47749914],
           [ 0.76329604, 0.39514048, 0.4741975 ],
           [ 0.76916794, 0.39702603, 0.47081439],
           [ 0.77502298, 0.39893446, 0.46735   ],
           [ 0.78085877, 0.400868  , 0.46380463],
           [ 0.78667286, 0.40282892, 0.46017874],
           [ 0.79246271, 0.4048196 , 0.45647292],
           [ 0.79822569, 0.40684249, 0.45268792],
           [ 0.80395908, 0.40890011, 0.44882462],
           [ 0.80966007, 0.41099506, 0.44488409],
           [ 0.81532584, 0.41313001, 0.4408672 ],
           [ 0.82095338, 0.41530771, 0.43677537],
           [ 0.82653946, 0.41753108, 0.43261066],
           [ 0.83208085, 0.41980305, 0.42837489],
           [ 0.83757423, 0.42212661, 0.42407011],
           [ 0.84301618, 0.42450482, 0.4196986 ],
           [ 0.84840316, 0.42694082, 0.41526288],
           [ 0.85373155, 0.4294378 , 0.41076573],
           [ 0.8589976 , 0.43199897, 0.40621024],
           [ 0.8641975 , 0.4346276 , 0.40159975],
           [ 0.86932733, 0.43732696, 0.39693796],
           [ 0.87438311, 0.44010033, 0.39222887],
           [ 0.87936081, 0.44295096, 0.38747682],
           [ 0.88425632, 0.44588207, 0.38268654],
           [ 0.88906554, 0.44889681, 0.37786308],
           [ 0.89378437, 0.45199824, 0.3730116 ],
           [ 0.89840867, 0.45518929, 0.36813827],
           [ 0.90293439, 0.45847274, 0.36324942],
           [ 0.90735756, 0.46185117, 0.35835166],
           [ 0.91167433, 0.46532694, 0.35345197],
           [ 0.915881  , 0.46890217, 0.34855767],
           [ 0.91997406, 0.47257868, 0.34367638],
           [ 0.92395023, 0.47635796, 0.33881601],
           [ 0.92780648, 0.48024117, 0.33398471],
           [ 0.93154008, 0.48422909, 0.3291908 ],
           [ 0.93514863, 0.48832213, 0.32444275],
           [ 0.93863008, 0.49252027, 0.31974912],
           [ 0.94198274, 0.49682311, 0.31511847],
           [ 0.94520535, 0.50122981, 0.31055931],
           [ 0.94829701, 0.50573917, 0.30608003],
           [ 0.95125725, 0.51034957, 0.30168887],
           [ 0.954086  , 0.51505906, 0.29739377],
           [ 0.95678355, 0.51986536, 0.2932024 ],
           [ 0.95935057, 0.52476587, 0.28912206],
           [ 0.96178805, 0.52975777, 0.28515964],
           [ 0.9640973 , 0.53483802, 0.28132156],
           [ 0.96627986, 0.54000343, 0.27761379],
           [ 0.9683375 , 0.5452507 , 0.27404177],
           [ 0.97027215, 0.55057645, 0.27061043],
           [ 0.97208896, 0.55597442, 0.26733231],
           [ 0.97378824, 0.56144311, 0.26420514],
           [ 0.97537225, 0.56697917, 0.2612321 ],
           [ 0.97684333, 0.5725793 , 0.2584161 ],
           [ 0.97820376, 0.57824032, 0.25575953],
           [ 0.9794557 , 0.58395922, 0.25326432],
           [ 0.9806012 , 0.58973318, 0.25093197],
           [ 0.98164451, 0.59555764, 0.24876746],
           [ 0.9825958 , 0.60142353, 0.24678389],
           [ 0.98344841, 0.60733543, 0.24496692],
           [ 0.98420378, 0.61329131, 0.24331643],
           [ 0.98486309, 0.61928934, 0.24183204],
           [ 0.98543359, 0.62532321, 0.24052074],
           [ 0.98592661, 0.63138392, 0.2393925 ],
           [ 0.98632908, 0.63748058, 0.2384285 ],
           [ 0.98664126, 0.64361223, 0.23762727],
           [ 0.9868743 , 0.64977022, 0.23699778],
           [ 0.98703872, 0.65594684, 0.236546  ],
           [ 0.98711602, 0.66215449, 0.23625164],
           [ 0.98710548, 0.66839291, 0.23611251],
           [ 0.98704522, 0.67463615, 0.23615516],
           [ 0.98690081, 0.68090692, 0.23634896],
           [ 0.98666929, 0.68720645, 0.23669008],
           [ 0.98639295, 0.6935072 , 0.23720252],
           [ 0.98603581, 0.69983201, 0.23785913],
           [ 0.98559369, 0.70618283, 0.23865535],
           [ 0.98511668, 0.71252856, 0.23961378],
           [ 0.98455154, 0.71890163, 0.24070275],
           [ 0.98392192, 0.72528731, 0.2419305 ],
           [ 0.98324116, 0.73167761, 0.24329811],
           [ 0.98246864, 0.738096  , 0.2447854 ],
           [ 0.98166755, 0.74450581, 0.24641186],
           [ 0.98078113, 0.75093941, 0.24815276],
           [ 0.97983772, 0.75738047, 0.25001371],
           [ 0.97883927, 0.7638279 , 0.25199054],
           [ 0.97775747, 0.7702969 , 0.25407166],
           [ 0.97664794, 0.77675738, 0.25626641],
           [ 0.9754357 , 0.78324913, 0.25855344],
           [ 0.9742107 , 0.7897247 , 0.26094721],
           [ 0.97288357, 0.79623051, 0.2634262 ],
           [ 0.97152974, 0.80272764, 0.26599948],
           [ 0.97008613, 0.80924798, 0.2686529 ],
           [ 0.96860571, 0.81576486, 0.27138993],
           [ 0.96704307, 0.82230061, 0.27420073],
           [ 0.96543772, 0.82883583, 0.27708584],
           [ 0.96375242, 0.83538829, 0.28003763],
           [ 0.96202315, 0.84194079, 0.28305544],
           [ 0.96021049, 0.84851164, 0.28613254],
           [ 0.95835763, 0.85508072, 0.28926815],
           [ 0.95641175, 0.86167196, 0.29245571],
           [ 0.95443484, 0.8682573 , 0.29569479],
           [ 0.95234871, 0.87487125, 0.29897882],
           [ 0.95024646, 0.88147281, 0.30230776],
           [ 0.94801413, 0.88811109, 0.3056752 ],
           [ 0.94578198, 0.89473018, 0.30908112],
           [ 0.94341986, 0.90138491, 0.31252002],
           [ 0.94102862, 0.90803285, 0.31599064],
           [ 0.93853313, 0.91470475, 0.31948933],
           [ 0.93597111, 0.92138481, 0.32301384],
           [ 0.9333376 , 0.92807483, 0.32656155],
           [ 0.93059157, 0.93479051, 0.33012998],
           [ 0.92781425, 0.94149986, 0.33371683],
           [ 0.92488992, 0.94824702, 0.33731997],
           [ 0.92194114, 0.95498491, 0.34093705],
           [ 0.91886139, 0.96175325, 0.3445663 ],
           [ 0.91569318, 0.96853549, 0.34820569],
           [ 0.91244907, 0.97532669, 0.35185336],
           [ 0.90904184, 0.98215741, 0.35550781]]



import madmom
import numpy as np
import webbrowser
import time
from codelab_adapter_client import AdapterNode


# EIM 初始化
class MyNode(AdapterNode):
    NODE_ID = "eim/spectrogram_thermal"

    def __init__(self):
        super().__init__()
    
    def send_data(self, content):
        message = self.message_template()
        message["payload"]["content"] = content
        self.publish(message)


node = MyNode()
node.receive_loop_as_thread()
time.sleep(0.1)

   

# 使用 Madmom 实时抓取音频流数据,做 sftf 并生成 sepctrogram
kwargs = dict(
    sample_rate=44100,                        # 采样率
    num_channels=1,                           # 单声道
    frame_size=2048,                          # 以 2048 sample 为一个 frame 做 stft 分析,同时也是窗口大小
    hop_size=441,                             # 间隔 441 sample 有重合的取 frame 做 stft 分析
    filterbank=madmom.audio.filters.MelFilterbank,          # 对频率做 MelFilter
    num_bands=12,                             # 分为 12 个频段
    #dtype=np.int16
    
    
)


stream=madmom.audio.signal.Stream(**kwargs)

#spec_list = [i for i in range(10)]            # 以 10 个时间点的结果为单位向 Scratch 发送列表数据

n = 0

color_resolution = 2/256                       # spectrogram 最大值不超过 2,取值范围受不同输入设备影响,最好后面加一个开始前测试最大音量的部分


spec_color=[i for i in range(10)]


try:
    for frames in stream:  
        fs = madmom.audio.signal.FramedSignal(frames,**kwargs)   # 将以 sample 为单位的信号转为以 frame 为单位的信号
        stft = madmom.audio.stft.STFT(fs)                        # 做 stft 分析
        spec = madmom.audio.spectrogram.LogarithmicFilteredSpectrogram(stft, **kwargs)     # 对频率做 MelFilter 同时对强度值做对数转换
        i = n%2
        if i == 0:                                               # 累积两个 frame 的结果发送给 Scratch
            for c in range(len(spec)):
                rgb_column = [i for i in range(12)]
                for r in range(len(spec[c])):
                    cmap_index=int(np.round(spec[c][r]/color_resolution)-1)    # 这部分是在做数值与颜色的映射
                    if cmap_index < 0:                                         # 避免超出颜色取值区间,但是如果已提前获得设备的最大响度,这部分代码就不需要了
                        cmap_index = 0                                         # 这里主要是先保证程序能持续运行
                    if cmap_index > 255:
                        cmap_index = 255
                    rgb_column[r]=thermal_rgb[cmap_index]
                spec_color[i*5+c]=rgb_column
            n = n + 1
        elif i == 1:
            for c in range(len(spec)):
                rgb_column = [i for i in range(12)]
                for r in range(len(spec[c])):
                    cmap_index=int(np.round(spec[c][r]/color_resolution)-1)
                    if cmap_index < 0:                                       
                        cmap_index = 0
                    if cmap_index > 255:
                        cmap_index = 255
                    rgb_column[r]=thermal_rgb[cmap_index]
                spec_color[i*5+c]=rgb_column
            node.send_data(spec_color)
            #print(spec_color)
            n = n + 1   
            

except KeyboardInterrupt:
    stream.close()
    print('interrupt by user')